diff --git a/.github/actions/ci-optimization/action.yml b/.github/actions/ci-optimization/action.yml index 0dcbdcecf34ad..2f677a0e552c2 100644 --- a/.github/actions/ci-optimization/action.yml +++ b/.github/actions/ci-optimization/action.yml @@ -17,6 +17,9 @@ outputs: ingestion-change: description: "Ingestion code has changed" value: ${{ steps.filter.outputs.ingestion == 'true' }} + ingestion-base-change: + description: "Ingestion base image docker image has changed" + value: ${{ steps.filter.outputs.ingestion-base == 'true' }} frontend-change: description: "Frontend code has changed" value: ${{ steps.filter.outputs.frontend == 'true' }} @@ -56,6 +59,8 @@ runs: - "metadata-models/**" - "smoke-test/**" - "docker/datahub-ingestion**" + ingestion-base: + - "docker/datahub-ingestion-base/**" docker: - "docker/**" backend: diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml index a9be56a69560c..2c931162fb006 100644 --- a/.github/workflows/airflow-plugin.yml +++ b/.github/workflows/airflow-plugin.yml @@ -61,7 +61,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - uses: acryldata/sane-checkout-action@v3 - uses: actions/setup-python@v4 with: diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 3f37fffc599bb..c93267947b65a 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -67,7 +67,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - uses: actions/setup-python@v4 if: ${{ needs.setup.outputs.ingestion_change == 'true' }} with: diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml index ca3510217f74a..b45d130e0494d 100644 --- a/.github/workflows/check-datahub-jars.yml +++ b/.github/workflows/check-datahub-jars.yml @@ -33,7 +33,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - uses: actions/setup-python@v4 with: python-version: "3.10" diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index fe50ad67c6492..9487e71e8da3d 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -52,6 +52,7 @@ jobs: repository_name: ${{ steps.tag.outputs.repository_name }} frontend_change: ${{ steps.ci-optimize.outputs.frontend-change == 'true' }} ingestion_change: ${{ steps.ci-optimize.outputs.ingestion-change == 'true' }} + ingestion_base_change: ${{ steps.ci-optimize.outputs.ingestion-base-change == 'true' }} backend_change: ${{ steps.ci-optimize.outputs.backend-change == 'true' }} frontend_only: ${{ steps.ci-optimize.outputs.frontend-only == 'true' }} ingestion_only: ${{ steps.ci-optimize.outputs.ingestion-only == 'true' }} @@ -125,7 +126,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - name: Run lint on smoke test if: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }} run: | @@ -144,7 +145,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - name: Check out the repo uses: acryldata/sane-checkout-action@v3 - name: Pre-build artifacts for docker image @@ -208,7 +209,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - name: Check out the repo uses: acryldata/sane-checkout-action@v3 - name: Pre-build artifacts for docker image @@ -272,7 +273,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - name: Check out the repo uses: acryldata/sane-checkout-action@v3 - name: Pre-build artifacts for docker image @@ -336,7 +337,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - name: Check out the repo uses: acryldata/sane-checkout-action@v3 - name: Pre-build artifacts for docker image @@ -400,7 +401,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - name: Check out the repo uses: acryldata/sane-checkout-action@v3 - name: Pre-build artifacts for docker image @@ -528,14 +529,8 @@ jobs: steps: - name: Check out the repo uses: acryldata/sane-checkout-action@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - datahub-ingestion-base: - - 'docker/datahub-ingestion-base/**' - name: Build and push Base Image - if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' }} + if: ${{ needs.setup.outputs.ingestion_base_change == 'true' }} uses: ./.github/actions/docker-custom-build-and-push with: target: base @@ -550,7 +545,7 @@ jobs: platforms: linux/amd64,linux/arm64/v8 - name: Compute DataHub Ingestion (Base) Tag id: tag - run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }}" >> $GITHUB_OUTPUT + run: echo "tag=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }}" >> $GITHUB_OUTPUT datahub_ingestion_base_slim_build: name: Build and Push DataHub Ingestion (Base-Slim) Docker Image runs-on: ubuntu-latest @@ -561,25 +556,19 @@ jobs: steps: - name: Check out the repo uses: acryldata/sane-checkout-action@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - datahub-ingestion-base: - - 'docker/datahub-ingestion-base/**' - name: Download Base Image uses: ishworkh/docker-image-artifact-download@v1 - if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }} + if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' && needs.setup.outputs.ingestion_base_change == 'true' }} with: - image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }} + image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }} - name: Login to DockerHub uses: docker/login-action@v3 - if: ${{ needs.setup.outputs.docker-login == 'true' && needs.setup.outputs.publish == 'false' && needs.setup.outputs.pr-publish == 'false' && steps.filter.outputs.datahub-ingestion-base == 'false' }} + if: ${{ needs.setup.outputs.docker-login == 'true' && needs.setup.outputs.publish == 'false' && needs.setup.outputs.pr-publish == 'false' && needs.setup.outputs.ingestion_base_change == 'false' }} with: username: ${{ secrets.ACRYL_DOCKER_USERNAME }} password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - name: Build and push Base-Slim Image - if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' }} + if: ${{ needs.setup.outputs.ingestion_base_change == 'true' }} uses: ./.github/actions/docker-custom-build-and-push with: target: slim-install @@ -590,14 +579,14 @@ jobs: password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} build-args: | APP_ENV=slim - BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }} + BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }} publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} context: . file: ./docker/datahub-ingestion-base/Dockerfile platforms: linux/amd64,linux/arm64/v8 - name: Compute DataHub Ingestion (Base-Slim) Tag id: tag - run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT + run: echo "tag=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT datahub_ingestion_base_full_build: name: Build and Push DataHub Ingestion (Base-Full) Docker Image runs-on: ubuntu-latest @@ -608,25 +597,19 @@ jobs: steps: - name: Check out the repo uses: acryldata/sane-checkout-action@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - datahub-ingestion-base: - - 'docker/datahub-ingestion-base/**' - name: Download Base Image uses: ishworkh/docker-image-artifact-download@v1 - if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }} + if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' && needs.setup.outputs.ingestion_base_change == 'true' }} with: - image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }} + image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }} - name: Login to DockerHub uses: docker/login-action@v3 - if: ${{ needs.setup.outputs.docker-login == 'true' && needs.setup.outputs.publish == 'false' && needs.setup.outputs.pr-publish == 'false' && steps.filter.outputs.datahub-ingestion-base == 'false' }} + if: ${{ needs.setup.outputs.docker-login == 'true' && needs.setup.outputs.publish == 'false' && needs.setup.outputs.pr-publish == 'false' && needs.setup.outputs.ingestion_base_change == 'false' }} with: username: ${{ secrets.ACRYL_DOCKER_USERNAME }} password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - name: Build and push (Base-Full) Image - if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' }} + if: ${{ needs.setup.outputs.ingestion_base_change == 'true' }} uses: ./.github/actions/docker-custom-build-and-push with: target: full-install @@ -637,21 +620,21 @@ jobs: password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} build-args: | APP_ENV=full - BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }} + BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }} publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} context: . file: ./docker/datahub-ingestion-base/Dockerfile platforms: linux/amd64,linux/arm64/v8 - name: Compute DataHub Ingestion (Base-Full) Tag id: tag - run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_full_tag || 'head' }}" >> $GITHUB_OUTPUT + run: echo "tag=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_full_tag || 'head' }}" >> $GITHUB_OUTPUT datahub_ingestion_slim_build: name: Build and Push DataHub Ingestion Docker Images runs-on: ubuntu-latest outputs: tag: ${{ steps.tag.outputs.tag }} - needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true') }} + needs_artifact_download: ${{ needs.setup.outputs.ingestion_change == 'true' && ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true') }} needs: [setup, datahub_ingestion_base_slim_build] if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: @@ -660,33 +643,25 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - name: Check out the repo uses: acryldata/sane-checkout-action@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - datahub-ingestion-base: - - 'docker/datahub-ingestion-base/**' - datahub-ingestion: - - 'docker/datahub-ingestion/**' - name: Build codegen - if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }} + if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} run: ./gradlew :metadata-ingestion:codegen - name: Download Base Image uses: ishworkh/docker-image-artifact-download@v1 - if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }} + if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' && needs.setup.outputs.ingestion_base_change == 'true' }} with: - image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }} + image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }} - name: Login to DockerHub uses: docker/login-action@v3 - if: ${{ needs.setup.outputs.docker-login == 'true' && needs.setup.outputs.publish == 'false' && needs.setup.outputs.pr-publish == 'false' && steps.filter.outputs.datahub-ingestion-base == 'false' }} + if: ${{ needs.setup.outputs.docker-login == 'true' && needs.setup.outputs.publish == 'false' && needs.setup.outputs.pr-publish == 'false' && needs.setup.outputs.ingestion_base_change == 'false' }} with: username: ${{ secrets.ACRYL_DOCKER_USERNAME }} password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - name: Build and push Slim Image - if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }} + if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} uses: ./.github/actions/docker-custom-build-and-push with: target: final @@ -694,7 +669,7 @@ jobs: ${{ env.DATAHUB_INGESTION_IMAGE }} build-args: | BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }} - DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }} + DOCKER_VERSION=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }} RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }} APP_ENV=slim tags: ${{ needs.setup.outputs.slim_tag }} @@ -706,7 +681,7 @@ jobs: platforms: linux/amd64,linux/arm64/v8 - name: Compute Tag id: tag - run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT + run: echo "tag=${{ needs.setup.outputs.ingestion_change == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT datahub_ingestion_slim_scan: permissions: contents: read # for actions/checkout to fetch code @@ -746,7 +721,7 @@ jobs: runs-on: ubuntu-latest outputs: tag: ${{ steps.tag.outputs.tag }} - needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' ) }} + needs_artifact_download: ${{ needs.setup.outputs.ingestion_change == 'true' && ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' ) }} needs: [setup, datahub_ingestion_base_full_build] if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: @@ -755,33 +730,25 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - name: Check out the repo uses: acryldata/sane-checkout-action@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - datahub-ingestion-base: - - 'docker/datahub-ingestion-base/**' - datahub-ingestion: - - 'docker/datahub-ingestion/**' - name: Build codegen - if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }} + if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} run: ./gradlew :metadata-ingestion:codegen - name: Download Base Image uses: ishworkh/docker-image-artifact-download@v1 - if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }} + if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' && needs.setup.outputs.ingestion_base_change == 'true' }} with: - image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }} + image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }} - name: Login to DockerHub uses: docker/login-action@v3 - if: ${{ needs.setup.outputs.docker-login == 'true' && needs.setup.outputs.publish == 'false' && needs.setup.outputs.pr-publish == 'false' && steps.filter.outputs.datahub-ingestion-base == 'false' }} + if: ${{ needs.setup.outputs.docker-login == 'true' && needs.setup.outputs.publish == 'false' && needs.setup.outputs.pr-publish == 'false' && needs.setup.outputs.ingestion_base_change == 'false' }} with: username: ${{ secrets.ACRYL_DOCKER_USERNAME }} password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - name: Build and push Full Image - if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }} + if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} uses: ./.github/actions/docker-custom-build-and-push with: target: final @@ -789,7 +756,7 @@ jobs: ${{ env.DATAHUB_INGESTION_IMAGE }} build-args: | BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }} - DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }} + DOCKER_VERSION=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }} RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }} tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.ACRYL_DOCKER_USERNAME }} @@ -800,7 +767,7 @@ jobs: platforms: linux/amd64,linux/arm64/v8 - name: Compute Tag (Full) id: tag - run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_tag || 'head' }}" >> $GITHUB_OUTPUT + run: echo "tag=${{ needs.setup.outputs.ingestion_change == 'true' && needs.setup.outputs.unique_tag || 'head' }}" >> $GITHUB_OUTPUT datahub_ingestion_full_scan: permissions: contents: read # for actions/checkout to fetch code @@ -890,7 +857,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - uses: actions/setup-python@v4 with: python-version: "3.10" diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index ab7890ad5b44f..99c1ad466b669 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -38,7 +38,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - uses: actions/setup-python@v4 with: python-version: "3.10" diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml index c978a8ff09b5e..51b97552eb150 100644 --- a/.github/workflows/metadata-ingestion.yml +++ b/.github/workflows/metadata-ingestion.yml @@ -51,7 +51,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - uses: acryldata/sane-checkout-action@v3 - uses: actions/setup-python@v4 with: diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml index c59675cf00614..6797c7ad67c0b 100644 --- a/.github/workflows/metadata-io.yml +++ b/.github/workflows/metadata-io.yml @@ -53,7 +53,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - uses: actions/setup-python@v4 if: ${{ needs.setup.outputs.ingestion_change == 'true' }} with: diff --git a/.github/workflows/metadata-model.yml b/.github/workflows/metadata-model.yml index 239d2d991915c..558b7c80f727c 100644 --- a/.github/workflows/metadata-model.yml +++ b/.github/workflows/metadata-model.yml @@ -34,7 +34,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - uses: acryldata/sane-checkout-action@v3 - uses: actions/setup-python@v4 with: diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 4fdb4a5b92e4d..77874abedaabd 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -40,12 +40,12 @@ jobs: "RyanHolstien", "Kunal-kankriya", "purnimagarg1", - "gaurav2733", "dushayntAW", - "AvaniSiddhapuraAPT", - "akarsh991", - "shubhamjagtap639", - "mayurinehate" + "sagar-salvi-apptware", + "kushagra-apptware", + "Salman-Apptware", + "mayurinehate", + "noggi" ]'), github.actor ) diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml index 46421c8e5b8eb..7137302c73564 100644 --- a/.github/workflows/publish-datahub-jars.yml +++ b/.github/workflows/publish-datahub-jars.yml @@ -54,7 +54,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - uses: actions/setup-python@v4 with: python-version: "3.10" @@ -167,3 +167,29 @@ jobs: echo signingKey=$SIGNING_KEY >> gradle.properties ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-auth:auth-api:publish ./gradlew :metadata-auth:auth-api:closeAndReleaseRepository --info + - name: publish datahub-custom-plugin-lib snapshot jar + if: ${{ github.event_name != 'release' }} + env: + RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} + RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} + SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} + SIGNING_KEY: ${{ secrets.SIGNING_KEY }} + NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} + NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + run: | + echo signingKey=$SIGNING_KEY >> gradle.properties + ./gradlew :metadata-integration:java:custom-plugin-lib:printVersion + ./gradlew :metadata-integration:java:custom-plugin-lib:publish + - name: release datahub-custom-plugin-lib jar + if: ${{ github.event_name == 'release' }} + env: + RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} + RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} + SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} + SIGNING_KEY: ${{ secrets.SIGNING_KEY }} + NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} + NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + run: | + echo signingKey=$SIGNING_KEY >> gradle.properties + ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-integration:java:custom-plugin-lib:publish + ./gradlew :metadata-integration:java:custom-plugin-lib:closeAndReleaseRepository --info diff --git a/.github/workflows/spark-smoke-test.yml b/.github/workflows/spark-smoke-test.yml index d143906bc7ccb..8ffc8420ba941 100644 --- a/.github/workflows/spark-smoke-test.yml +++ b/.github/workflows/spark-smoke-test.yml @@ -35,7 +35,7 @@ jobs: with: distribution: "zulu" java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/actions/setup-gradle@v3 - uses: actions/setup-python@v4 with: python-version: "3.10" diff --git a/.gitignore b/.gitignore index 1fcca8751131f..43c627f9ed244 100644 --- a/.gitignore +++ b/.gitignore @@ -126,3 +126,6 @@ metadata-service/war/bin/ metadata-utils/bin/ test-models/bin/ +datahub-executor/ +datahub-integrations-service/ +metadata-ingestion-modules/acryl-cloud diff --git a/README.md b/README.md index 15289f663f7a8..b3c2e2d545941 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ Here are the companies that have officially adopted DataHub. Please feel free to - [Cabify](https://cabify.tech/) - [ClassDojo](https://www.classdojo.com/) - [Coursera](https://www.coursera.org/) +- [CVS Health](https://www.cvshealth.com/) - [DefinedCrowd](http://www.definedcrowd.com) - [DFDS](https://www.dfds.com/) - [Digital Turbine](https://www.digitalturbine.com/) diff --git a/build.gradle b/build.gradle index 437ac4fbf6f32..569987cf73c6c 100644 --- a/build.gradle +++ b/build.gradle @@ -32,11 +32,11 @@ buildscript { ext.junitJupiterVersion = '5.6.1' // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md - ext.pegasusVersion = '29.51.6' + ext.pegasusVersion = '29.57.0' ext.mavenVersion = '3.6.3' - ext.springVersion = '6.1.4' - ext.springBootVersion = '3.2.3' - ext.springKafkaVersion = '3.1.2' + ext.springVersion = '6.1.5' + ext.springBootVersion = '3.2.6' + ext.springKafkaVersion = '3.1.6' ext.openTelemetryVersion = '1.18.0' ext.neo4jVersion = '5.14.0' ext.neo4jTestVersion = '5.14.0' @@ -44,9 +44,9 @@ buildscript { ext.testContainersVersion = '1.17.4' ext.elasticsearchVersion = '2.11.1' // ES 7.10, Opensearch 1.x, 2.x ext.jacksonVersion = '2.15.3' - ext.jettyVersion = '11.0.19' - ext.playVersion = '2.8.21' - ext.log4jVersion = '2.19.0' + ext.jettyVersion = '11.0.21' + ext.playVersion = '2.8.22' + ext.log4jVersion = '2.23.1' ext.slf4jVersion = '1.7.36' ext.logbackClassic = '1.4.14' ext.hadoop3Version = '3.3.5' @@ -54,7 +54,7 @@ buildscript { ext.hazelcastVersion = '5.3.6' ext.ebeanVersion = '12.16.1' ext.googleJavaFormatVersion = '1.18.1' - ext.openLineageVersion = '1.14.0' + ext.openLineageVersion = '1.16.0' ext.logbackClassicJava8 = '1.2.12' ext.docker_registry = 'acryldata' @@ -149,7 +149,6 @@ project.ext.externalDependency = [ 'hazelcastTest':"com.hazelcast:hazelcast:$hazelcastVersion:tests", 'hibernateCore': 'org.hibernate:hibernate-core:5.2.16.Final', 'httpClient': 'org.apache.httpcomponents.client5:httpclient5:5.3', - 'httpAsyncClient': 'org.apache.httpcomponents:httpasyncclient:4.1.5', 'iStackCommons': 'com.sun.istack:istack-commons-runtime:4.0.1', 'jacksonJDK8': "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$jacksonVersion", 'jacksonDataPropertyFormat': "com.fasterxml.jackson.dataformat:jackson-dataformat-properties:$jacksonVersion", @@ -250,6 +249,7 @@ project.ext.externalDependency = [ 'springBootStarterValidation': "org.springframework.boot:spring-boot-starter-validation:$springBootVersion", 'springKafka': "org.springframework.kafka:spring-kafka:$springKafkaVersion", 'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion", + 'springRetry': "org.springframework.retry:spring-retry:2.0.6", 'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15', 'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46', 'swaggerCore': 'io.swagger.core.v3:swagger-core:2.2.7', diff --git a/datahub-frontend/app/auth/AuthModule.java b/datahub-frontend/app/auth/AuthModule.java index 7db8f5689ead5..32dfba00d47db 100644 --- a/datahub-frontend/app/auth/AuthModule.java +++ b/datahub-frontend/app/auth/AuthModule.java @@ -63,6 +63,7 @@ public class AuthModule extends AbstractModule { private static final String ENTITY_CLIENT_RETRY_INTERVAL = "entityClient.retryInterval"; private static final String ENTITY_CLIENT_NUM_RETRIES = "entityClient.numRetries"; private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE = "entityClient.restli.get.batchSize"; + private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY = "entityClient.restli.get.batchConcurrency"; private static final String GET_SSO_SETTINGS_ENDPOINT = "auth/getSsoSettings"; private final com.typesafe.config.Config _configs; @@ -208,7 +209,8 @@ protected SystemEntityClient provideEntityClient( new ExponentialBackoff(_configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)), _configs.getInt(ENTITY_CLIENT_NUM_RETRIES), configurationProvider.getCache().getClient().getEntityClient(), - Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE))); + Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE)), + Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY))); } @Provides diff --git a/datahub-frontend/app/controllers/Application.java b/datahub-frontend/app/controllers/Application.java index df0cd4f4ff82f..d17e600aadc07 100644 --- a/datahub-frontend/app/controllers/Application.java +++ b/datahub-frontend/app/controllers/Application.java @@ -155,7 +155,7 @@ AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER, getDataHubActorHeader(req .setBody( new InMemoryBodyWritable( ByteString.fromByteBuffer(request.body().asBytes().asByteBuffer()), - "application/json")) + request.contentType().orElse("application/json"))) .setRequestTimeout(Duration.ofSeconds(120)) .execute() .thenApply( diff --git a/datahub-frontend/conf/application.conf b/datahub-frontend/conf/application.conf index 6aa58d5b13b2c..dc243ecadafd8 100644 --- a/datahub-frontend/conf/application.conf +++ b/datahub-frontend/conf/application.conf @@ -38,8 +38,12 @@ jwt { play.server.provider = server.CustomAkkaHttpServerProvider play.http.server.akka.max-header-count = 64 play.http.server.akka.max-header-count = ${?DATAHUB_AKKA_MAX_HEADER_COUNT} -play.server.akka.max-header-size = 8k +# max-header-size is reportedly no longer used +play.server.akka.max-header-size = 32k play.server.akka.max-header-size = ${?DATAHUB_AKKA_MAX_HEADER_VALUE_LENGTH} +# max header value length seems to impact the actual limit +play.server.akka.max-header-value-length = 32k +play.server.akka.max-header-value-length = ${?DATAHUB_AKKA_MAX_HEADER_VALUE_LENGTH} # Update AUTH_COOKIE_SAME_SITE and AUTH_COOKIE_SECURE in order to change how authentication cookies # are configured. If you wish cookies to be sent in first and third party contexts, set @@ -289,5 +293,7 @@ entityClient.retryInterval = 2 entityClient.retryInterval = ${?ENTITY_CLIENT_RETRY_INTERVAL} entityClient.numRetries = 3 entityClient.numRetries = ${?ENTITY_CLIENT_NUM_RETRIES} -entityClient.restli.get.batchSize = 100 -entityClient.restli.get.batchSize = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE} \ No newline at end of file +entityClient.restli.get.batchSize = 50 +entityClient.restli.get.batchSize = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE} +entityClient.restli.get.batchConcurrency = 2 +entityClient.restli.get.batchConcurrency = ${?ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java index 0924dbc0c0a6d..f70c46ba943a5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java @@ -1,5 +1,8 @@ package com.linkedin.datahub.graphql; +import com.google.common.collect.ImmutableSet; +import java.util.Set; + /** Constants relating to GraphQL type system & execution. */ public class Constants { @@ -20,10 +23,19 @@ private Constants() {} public static final String LINEAGE_SCHEMA_FILE = "lineage.graphql"; public static final String PROPERTIES_SCHEMA_FILE = "properties.graphql"; public static final String FORMS_SCHEMA_FILE = "forms.graphql"; + public static final String ASSERTIONS_SCHEMA_FILE = "assertions.graphql"; public static final String INCIDENTS_SCHEMA_FILE = "incident.graphql"; + public static final String CONTRACTS_SCHEMA_FILE = "contract.graphql"; public static final String CONNECTIONS_SCHEMA_FILE = "connection.graphql"; public static final String BROWSE_PATH_DELIMITER = "/"; public static final String BROWSE_PATH_V2_DELIMITER = "␟"; public static final String VERSION_STAMP_FIELD_NAME = "versionStamp"; public static final String ENTITY_FILTER_NAME = "_entityType"; + + public static final Set DEFAULT_PERSONA_URNS = + ImmutableSet.of( + "urn:li:dataHubPersona:technicalUser", + "urn:li:dataHubPersona:businessUser", + "urn:li:dataHubPersona:dataLeader", + "urn:li:dataHubPersona:dataSteward"); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 5315a444d07b7..b751849859be4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -42,6 +42,7 @@ import com.linkedin.datahub.graphql.generated.CorpGroup; import com.linkedin.datahub.graphql.generated.CorpGroupInfo; import com.linkedin.datahub.graphql.generated.CorpUser; +import com.linkedin.datahub.graphql.generated.CorpUserEditableProperties; import com.linkedin.datahub.graphql.generated.CorpUserInfo; import com.linkedin.datahub.graphql.generated.CorpUserViewsSettings; import com.linkedin.datahub.graphql.generated.Dashboard; @@ -53,9 +54,12 @@ import com.linkedin.datahub.graphql.generated.DataHubView; import com.linkedin.datahub.graphql.generated.DataJob; import com.linkedin.datahub.graphql.generated.DataJobInputOutput; +import com.linkedin.datahub.graphql.generated.DataPlatform; import com.linkedin.datahub.graphql.generated.DataPlatformInstance; +import com.linkedin.datahub.graphql.generated.DataQualityContract; import com.linkedin.datahub.graphql.generated.Dataset; import com.linkedin.datahub.graphql.generated.DatasetStatsSummary; +import com.linkedin.datahub.graphql.generated.Deprecation; import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.ERModelRelationship; import com.linkedin.datahub.graphql.generated.ERModelRelationshipProperties; @@ -64,6 +68,7 @@ import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy; import com.linkedin.datahub.graphql.generated.ForeignKeyConstraint; import com.linkedin.datahub.graphql.generated.FormActorAssignment; +import com.linkedin.datahub.graphql.generated.FreshnessContract; import com.linkedin.datahub.graphql.generated.GetRootGlossaryNodesResult; import com.linkedin.datahub.graphql.generated.GetRootGlossaryTermsResult; import com.linkedin.datahub.graphql.generated.GlossaryNode; @@ -102,6 +107,7 @@ import com.linkedin.datahub.graphql.generated.QuickFilter; import com.linkedin.datahub.graphql.generated.RecommendationContent; import com.linkedin.datahub.graphql.generated.ResolvedAuditStamp; +import com.linkedin.datahub.graphql.generated.SchemaContract; import com.linkedin.datahub.graphql.generated.SchemaField; import com.linkedin.datahub.graphql.generated.SchemaFieldEntity; import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResult; @@ -118,7 +124,14 @@ import com.linkedin.datahub.graphql.resolvers.assertion.AssertionRunEventResolver; import com.linkedin.datahub.graphql.resolvers.assertion.DeleteAssertionResolver; import com.linkedin.datahub.graphql.resolvers.assertion.EntityAssertionsResolver; -import com.linkedin.datahub.graphql.resolvers.auth.*; +import com.linkedin.datahub.graphql.resolvers.assertion.ReportAssertionResultResolver; +import com.linkedin.datahub.graphql.resolvers.assertion.UpsertCustomAssertionResolver; +import com.linkedin.datahub.graphql.resolvers.auth.CreateAccessTokenResolver; +import com.linkedin.datahub.graphql.resolvers.auth.DebugAccessResolver; +import com.linkedin.datahub.graphql.resolvers.auth.GetAccessTokenMetadataResolver; +import com.linkedin.datahub.graphql.resolvers.auth.GetAccessTokenResolver; +import com.linkedin.datahub.graphql.resolvers.auth.ListAccessTokensResolver; +import com.linkedin.datahub.graphql.resolvers.auth.RevokeAccessTokenResolver; import com.linkedin.datahub.graphql.resolvers.browse.BrowsePathsResolver; import com.linkedin.datahub.graphql.resolvers.browse.BrowseResolver; import com.linkedin.datahub.graphql.resolvers.browse.EntityBrowsePathsResolver; @@ -136,6 +149,8 @@ import com.linkedin.datahub.graphql.resolvers.container.ParentContainersResolver; import com.linkedin.datahub.graphql.resolvers.dashboard.DashboardStatsSummaryResolver; import com.linkedin.datahub.graphql.resolvers.dashboard.DashboardUsageStatsResolver; +import com.linkedin.datahub.graphql.resolvers.datacontract.EntityDataContractResolver; +import com.linkedin.datahub.graphql.resolvers.datacontract.UpsertDataContractResolver; import com.linkedin.datahub.graphql.resolvers.dataproduct.BatchSetDataProductResolver; import com.linkedin.datahub.graphql.resolvers.dataproduct.CreateDataProductResolver; import com.linkedin.datahub.graphql.resolvers.dataproduct.DeleteDataProductResolver; @@ -158,8 +173,11 @@ import com.linkedin.datahub.graphql.resolvers.form.BatchAssignFormResolver; import com.linkedin.datahub.graphql.resolvers.form.BatchRemoveFormResolver; import com.linkedin.datahub.graphql.resolvers.form.CreateDynamicFormAssignmentResolver; +import com.linkedin.datahub.graphql.resolvers.form.CreateFormResolver; +import com.linkedin.datahub.graphql.resolvers.form.DeleteFormResolver; import com.linkedin.datahub.graphql.resolvers.form.IsFormAssignedToMeResolver; import com.linkedin.datahub.graphql.resolvers.form.SubmitFormPromptResolver; +import com.linkedin.datahub.graphql.resolvers.form.UpdateFormResolver; import com.linkedin.datahub.graphql.resolvers.form.VerifyFormResolver; import com.linkedin.datahub.graphql.resolvers.glossary.AddRelatedTermsResolver; import com.linkedin.datahub.graphql.resolvers.glossary.CreateGlossaryNodeResolver; @@ -271,6 +289,9 @@ import com.linkedin.datahub.graphql.resolvers.settings.view.UpdateGlobalViewsSettingsResolver; import com.linkedin.datahub.graphql.resolvers.step.BatchGetStepStatesResolver; import com.linkedin.datahub.graphql.resolvers.step.BatchUpdateStepStatesResolver; +import com.linkedin.datahub.graphql.resolvers.structuredproperties.CreateStructuredPropertyResolver; +import com.linkedin.datahub.graphql.resolvers.structuredproperties.RemoveStructuredPropertiesResolver; +import com.linkedin.datahub.graphql.resolvers.structuredproperties.UpdateStructuredPropertyResolver; import com.linkedin.datahub.graphql.resolvers.structuredproperties.UpsertStructuredPropertiesResolver; import com.linkedin.datahub.graphql.resolvers.tag.CreateTagResolver; import com.linkedin.datahub.graphql.resolvers.tag.DeleteTagResolver; @@ -367,6 +388,7 @@ import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; import com.linkedin.metadata.recommendation.RecommendationsService; +import com.linkedin.metadata.service.AssertionService; import com.linkedin.metadata.service.BusinessAttributeService; import com.linkedin.metadata.service.DataProductService; import com.linkedin.metadata.service.ERModelRelationshipService; @@ -444,6 +466,7 @@ public class GmsGraphQLEngine { private final FormService formService; private final RestrictedService restrictedService; private ConnectionService connectionService; + private AssertionService assertionService; private final BusinessAttributeService businessAttributeService; private final FeatureFlags featureFlags; @@ -565,6 +588,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { this.formService = args.formService; this.restrictedService = args.restrictedService; this.connectionService = args.connectionService; + this.assertionService = args.assertionService; this.businessAttributeService = args.businessAttributeService; this.ingestionConfiguration = Objects.requireNonNull(args.ingestionConfiguration); @@ -626,48 +650,49 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { this.businessAttributeType = new BusinessAttributeType(entityClient); // Init Lists this.entityTypes = - ImmutableList.of( - datasetType, - roleType, - corpUserType, - corpGroupType, - dataPlatformType, - chartType, - dashboardType, - tagType, - mlModelType, - mlModelGroupType, - mlFeatureType, - mlFeatureTableType, - mlPrimaryKeyType, - dataFlowType, - dataJobType, - glossaryTermType, - glossaryNodeType, - connectionType, - containerType, - notebookType, - domainType, - assertionType, - versionedDatasetType, - dataPlatformInstanceType, - accessTokenMetadataType, - testType, - dataHubPolicyType, - dataHubRoleType, - schemaFieldType, - erModelRelationshipType, - dataHubViewType, - queryType, - dataProductType, - ownershipType, - structuredPropertyType, - dataTypeType, - entityTypeType, - formType, - incidentType, - restrictedType, - businessAttributeType); + new ArrayList<>( + ImmutableList.of( + datasetType, + roleType, + corpUserType, + corpGroupType, + dataPlatformType, + chartType, + dashboardType, + tagType, + mlModelType, + mlModelGroupType, + mlFeatureType, + mlFeatureTableType, + mlPrimaryKeyType, + dataFlowType, + dataJobType, + glossaryTermType, + glossaryNodeType, + connectionType, + containerType, + notebookType, + domainType, + assertionType, + versionedDatasetType, + dataPlatformInstanceType, + accessTokenMetadataType, + testType, + dataHubPolicyType, + dataHubRoleType, + schemaFieldType, + erModelRelationshipType, + dataHubViewType, + queryType, + dataProductType, + ownershipType, + structuredPropertyType, + dataTypeType, + entityTypeType, + formType, + incidentType, + restrictedType, + businessAttributeType)); this.loadableTypes = new ArrayList<>(entityTypes); // Extend loadable types with types from the plugins // This allows us to offer search and browse capabilities out of the box for those types @@ -741,6 +766,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configureDomainResolvers(builder); configureDataProductResolvers(builder); configureAssertionResolvers(builder); + configureContractResolvers(builder); configurePolicyResolvers(builder); configureDataProcessInstanceResolvers(builder); configureVersionedDatasetResolvers(builder); @@ -763,6 +789,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configureBusinessAttributeResolver(builder); configureBusinessAttributeAssociationResolver(builder); configureConnectionResolvers(builder); + configureDeprecationResolvers(builder); } private void configureOrganisationRoleResolvers(RuntimeWiring.Builder builder) { @@ -814,7 +841,9 @@ public GraphQLEngine.Builder builder() { .addSchema(fileBasedSchema(PROPERTIES_SCHEMA_FILE)) .addSchema(fileBasedSchema(FORMS_SCHEMA_FILE)) .addSchema(fileBasedSchema(CONNECTIONS_SCHEMA_FILE)) - .addSchema(fileBasedSchema(INCIDENTS_SCHEMA_FILE)); + .addSchema(fileBasedSchema(ASSERTIONS_SCHEMA_FILE)) + .addSchema(fileBasedSchema(INCIDENTS_SCHEMA_FILE)) + .addSchema(fileBasedSchema(CONTRACTS_SCHEMA_FILE)); for (GmsGraphQLPlugin plugin : this.graphQLPlugins) { List pluginSchemaFiles = plugin.getSchemaFiles(); @@ -1206,6 +1235,10 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { "createTestConnectionRequest", new CreateTestConnectionRequestResolver( this.entityClient, this.ingestionConfiguration)) + .dataFetcher( + "upsertCustomAssertion", new UpsertCustomAssertionResolver(assertionService)) + .dataFetcher( + "reportAssertionResult", new ReportAssertionResultResolver(assertionService)) .dataFetcher( "deleteAssertion", new DeleteAssertionResolver(this.entityClient, this.entityService)) @@ -1291,10 +1324,23 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { .dataFetcher( "upsertStructuredProperties", new UpsertStructuredPropertiesResolver(this.entityClient)) + .dataFetcher( + "removeStructuredProperties", + new RemoveStructuredPropertiesResolver(this.entityClient)) + .dataFetcher( + "createStructuredProperty", + new CreateStructuredPropertyResolver(this.entityClient)) + .dataFetcher( + "updateStructuredProperty", + new UpdateStructuredPropertyResolver(this.entityClient)) .dataFetcher("raiseIncident", new RaiseIncidentResolver(this.entityClient)) .dataFetcher( "updateIncidentStatus", - new UpdateIncidentStatusResolver(this.entityClient, this.entityService)); + new UpdateIncidentStatusResolver(this.entityClient, this.entityService)) + .dataFetcher( + "createForm", new CreateFormResolver(this.entityClient, this.formService)) + .dataFetcher("deleteForm", new DeleteFormResolver(this.entityClient)) + .dataFetcher("updateForm", new UpdateFormResolver(this.entityClient)); if (featureFlags.isBusinessAttributeEntityEnabled()) { typeWiring .dataFetcher( @@ -1800,6 +1846,18 @@ private void configureCorpUserResolvers(final RuntimeWiring.Builder builder) { new LoadableTypeResolver<>( corpUserType, (env) -> ((CorpUserInfo) env.getSource()).getManager().getUrn()))); + builder.type( + "CorpUserEditableProperties", + typeWiring -> + typeWiring.dataFetcher( + "platforms", + new LoadableTypeBatchResolver<>( + dataPlatformType, + (env) -> + ((CorpUserEditableProperties) env.getSource()) + .getPlatforms().stream() + .map(DataPlatform::getUrn) + .collect(Collectors.toList())))); } /** @@ -2709,6 +2767,59 @@ private void configureAssertionResolvers(final RuntimeWiring.Builder builder) { "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry))); } + private void configureContractResolvers(final RuntimeWiring.Builder builder) { + builder.type( + "Dataset", + typeWiring -> + typeWiring.dataFetcher( + "contract", new EntityDataContractResolver(this.entityClient, this.graphClient))); + builder.type( + "FreshnessContract", + typeWiring -> + typeWiring.dataFetcher( + "assertion", + new LoadableTypeResolver<>( + getAssertionType(), + (env) -> { + final FreshnessContract contract = env.getSource(); + return contract.getAssertion() != null + ? contract.getAssertion().getUrn() + : null; + }))); + builder.type( + "DataQualityContract", + typeWiring -> + typeWiring.dataFetcher( + "assertion", + new LoadableTypeResolver<>( + getAssertionType(), + (env) -> { + final DataQualityContract contract = env.getSource(); + return contract.getAssertion() != null + ? contract.getAssertion().getUrn() + : null; + }))); + builder.type( + "SchemaContract", + typeWiring -> + typeWiring.dataFetcher( + "assertion", + new LoadableTypeResolver<>( + getAssertionType(), + (env) -> { + final SchemaContract contract = env.getSource(); + return contract.getAssertion() != null + ? contract.getAssertion().getUrn() + : null; + }))); + builder.type( + "Mutation", + typeWiring -> + typeWiring.dataFetcher( + "upsertDataContract", + new UpsertDataContractResolver(this.entityClient, this.graphClient))); + } + private void configurePolicyResolvers(final RuntimeWiring.Builder builder) { // Register resolvers for "resolvedUsers" and "resolvedGroups" field of the Policy type. builder.type( @@ -3053,4 +3164,14 @@ private void configureConnectionResolvers(final RuntimeWiring.Builder builder) { : null; }))); } + + private void configureDeprecationResolvers(final RuntimeWiring.Builder builder) { + builder.type( + "Deprecation", + typeWiring -> + typeWiring.dataFetcher( + "actorEntity", + new EntityTypeResolver( + entityTypes, (env) -> ((Deprecation) env.getSource()).getActorEntity()))); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java index d4d4d592d6bca..f6ab3a603dbb7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java @@ -25,6 +25,7 @@ import com.linkedin.metadata.graph.SiblingGraphService; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.recommendation.RecommendationsService; +import com.linkedin.metadata.service.AssertionService; import com.linkedin.metadata.service.BusinessAttributeService; import com.linkedin.metadata.service.DataProductService; import com.linkedin.metadata.service.ERModelRelationshipService; @@ -86,6 +87,7 @@ public class GmsGraphQLEngineArgs { boolean graphQLQueryIntrospectionEnabled; BusinessAttributeService businessAttributeService; ConnectionService connectionService; + AssertionService assertionService; // any fork specific args should go below this line } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java index c16e436a7805c..b9b4e4f4ef292 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java @@ -5,6 +5,7 @@ import static com.linkedin.metadata.Constants.*; import static com.linkedin.metadata.authorization.ApiOperation.DELETE; import static com.linkedin.metadata.authorization.ApiOperation.MANAGE; +import static com.linkedin.metadata.authorization.PoliciesConfig.MANAGE_ACCESS_TOKENS; import com.datahub.authorization.AuthUtil; import com.datahub.authorization.ConjunctivePrivilegeGroup; @@ -52,9 +53,11 @@ public static boolean canManagePolicies(@Nonnull QueryContext context) { public static boolean canGeneratePersonalAccessToken(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - PoliciesConfig.GENERATE_PERSONAL_ACCESS_TOKENS_PRIVILEGE); + context.getAuthorizer(), + context.getActorUrn(), + PoliciesConfig.GENERATE_PERSONAL_ACCESS_TOKENS_PRIVILEGE) + || AuthUtil.isAuthorized( + context.getAuthorizer(), context.getActorUrn(), MANAGE_ACCESS_TOKENS); } public static boolean canManageTokens(@Nonnull QueryContext context) { @@ -381,6 +384,20 @@ public static T restrictEntity(@Nonnull Object entity, Class clazz) { } } + public static boolean canManageStructuredProperties(@Nonnull QueryContext context) { + return AuthUtil.isAuthorized( + context.getAuthorizer(), + context.getActorUrn(), + PoliciesConfig.MANAGE_STRUCTURED_PROPERTIES_PRIVILEGE); + } + + public static boolean canManageForms(@Nonnull QueryContext context) { + return AuthUtil.isAuthorized( + context.getAuthorizer(), + context.getActorUrn(), + PoliciesConfig.MANAGE_DOCUMENTATION_FORMS_PRIVILEGE); + } + public static boolean isAuthorized( @Nonnull Authorizer authorizer, @Nonnull String actor, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java index 2a4f75cf6055a..2a9af37a06ad9 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java @@ -20,4 +20,5 @@ public class FeatureFlags { private boolean nestedDomainsEnabled = false; private boolean schemaFieldEntityFetchEnabled = false; private boolean businessAttributeEntityEnabled = false; + private boolean dataContractsEnabled = false; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java index 74d9e7f8a8c57..542745e014862 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java @@ -15,6 +15,7 @@ import com.linkedin.datahub.graphql.generated.AndFilterInput; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.resolvers.search.SearchUtils; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; @@ -112,10 +113,11 @@ public static Map buildFacetFilters( return facetFilters; } - public static List criterionListFromAndFilter(List andFilters) { + public static List criterionListFromAndFilter( + List andFilters, @Nullable AspectRetriever aspectRetriever) { return andFilters != null && !andFilters.isEmpty() ? andFilters.stream() - .map(filter -> criterionFromFilter(filter)) + .map(filter -> criterionFromFilter(filter, aspectRetriever)) .collect(Collectors.toList()) : Collections.emptyList(); } @@ -124,13 +126,14 @@ public static List criterionListFromAndFilter(List // conjunctive criterion // arrays, rather than just one for the AND case. public static ConjunctiveCriterionArray buildConjunctiveCriterionArrayWithOr( - @Nonnull List orFilters) { + @Nonnull List orFilters, @Nullable AspectRetriever aspectRetriever) { return new ConjunctiveCriterionArray( orFilters.stream() .map( orFilter -> { CriterionArray andCriterionForOr = - new CriterionArray(criterionListFromAndFilter(orFilter.getAnd())); + new CriterionArray( + criterionListFromAndFilter(orFilter.getAnd(), aspectRetriever)); return new ConjunctiveCriterion().setAnd(andCriterionForOr); }) .collect(Collectors.toList())); @@ -138,7 +141,9 @@ public static ConjunctiveCriterionArray buildConjunctiveCriterionArrayWithOr( @Nullable public static Filter buildFilter( - @Nullable List andFilters, @Nullable List orFilters) { + @Nullable List andFilters, + @Nullable List orFilters, + @Nullable AspectRetriever aspectRetriever) { if ((andFilters == null || andFilters.isEmpty()) && (orFilters == null || orFilters.isEmpty())) { return null; @@ -147,30 +152,33 @@ public static Filter buildFilter( // Or filters are the new default. We will check them first. // If we have OR filters, we need to build a series of CriterionArrays if (orFilters != null && !orFilters.isEmpty()) { - return new Filter().setOr(buildConjunctiveCriterionArrayWithOr(orFilters)); + return new Filter().setOr(buildConjunctiveCriterionArrayWithOr(orFilters, aspectRetriever)); } // If or filters are not set, someone may be using the legacy and filters - final List andCriterions = criterionListFromAndFilter(andFilters); + final List andCriterions = criterionListFromAndFilter(andFilters, aspectRetriever); return new Filter() .setOr( new ConjunctiveCriterionArray( new ConjunctiveCriterion().setAnd(new CriterionArray(andCriterions)))); } - public static Criterion criterionFromFilter(final FacetFilterInput filter) { - return criterionFromFilter(filter, false); + public static Criterion criterionFromFilter( + final FacetFilterInput filter, @Nullable AspectRetriever aspectRetriever) { + return criterionFromFilter(filter, false, aspectRetriever); } // Translates a FacetFilterInput (graphql input class) into Criterion (our internal model) public static Criterion criterionFromFilter( - final FacetFilterInput filter, final Boolean skipKeywordSuffix) { + final FacetFilterInput filter, + final Boolean skipKeywordSuffix, + @Nullable AspectRetriever aspectRetriever) { Criterion result = new Criterion(); if (skipKeywordSuffix) { result.setField(filter.getField()); } else { - result.setField(getFilterField(filter.getField(), skipKeywordSuffix)); + result.setField(getFilterField(filter.getField(), skipKeywordSuffix, aspectRetriever)); } // `value` is deprecated in place of `values`- this is to support old query patterns. If values @@ -205,11 +213,13 @@ public static Criterion criterionFromFilter( } private static String getFilterField( - final String originalField, final boolean skipKeywordSuffix) { + final String originalField, + final boolean skipKeywordSuffix, + @Nullable AspectRetriever aspectRetriever) { if (KEYWORD_EXCLUDED_FILTERS.contains(originalField)) { return originalField; } - return ESUtils.toKeywordField(originalField, skipKeywordSuffix); + return ESUtils.toKeywordField(originalField, skipKeywordSuffix, aspectRetriever); } public static Filter buildFilterWithUrns(@Nonnull Set urns, @Nullable Filter inputFilters) { @@ -239,6 +249,9 @@ public static Filter viewFilter( return null; } DataHubViewInfo viewInfo = resolveView(opContext, viewService, UrnUtils.getUrn(viewUrn)); + if (viewInfo == null) { + return null; + } Filter result = SearchUtils.combineFilters(null, viewInfo.getDefinition().getFilter()); return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolver.java index 3ca78d643679b..0e9d2cea61141 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolver.java @@ -15,6 +15,7 @@ import com.linkedin.datahub.graphql.types.dataset.mappers.AssertionRunEventMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.EnvelopedAspect; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; @@ -67,7 +68,10 @@ public CompletableFuture get(DataFetchingEnvironment e maybeStartTimeMillis, maybeEndTimeMillis, maybeLimit, - buildFilter(maybeFilters, maybeStatus)); + buildFilter( + maybeFilters, + maybeStatus, + context.getOperationContext().getAspectRetriever())); // Step 2: Bind profiles into GraphQL strong types. List runEvents = @@ -98,6 +102,16 @@ public CompletableFuture get(DataFetchingEnvironment e && AssertionResultType.SUCCESS.equals( runEvent.getResult().getType())) .count())); + result.setErrored( + Math.toIntExact( + runEvents.stream() + .filter( + runEvent -> + AssertionRunStatus.COMPLETE.equals(runEvent.getStatus()) + && runEvent.getResult() != null + && AssertionResultType.ERROR.equals( + runEvent.getResult().getType())) + .count())); result.setRunEvents(runEvents); return result; } catch (RemoteInvocationException e) { @@ -110,7 +124,9 @@ public CompletableFuture get(DataFetchingEnvironment e @Nullable public static Filter buildFilter( - @Nullable FilterInput filtersInput, @Nullable final String status) { + @Nullable FilterInput filtersInput, + @Nullable final String status, + @Nullable AspectRetriever aspectRetriever) { if (filtersInput == null && status == null) { return null; } @@ -131,7 +147,7 @@ public static Filter buildFilter( .setAnd( new CriterionArray( facetFilters.stream() - .map(filter -> criterionFromFilter(filter, true)) + .map(filter -> criterionFromFilter(filter, true, aspectRetriever)) .collect(Collectors.toList()))))); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionUtils.java new file mode 100644 index 0000000000000..757ff38de6006 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionUtils.java @@ -0,0 +1,27 @@ +package com.linkedin.datahub.graphql.resolvers.assertion; + +import com.datahub.authorization.ConjunctivePrivilegeGroup; +import com.datahub.authorization.DisjunctivePrivilegeGroup; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.metadata.authorization.PoliciesConfig; + +public class AssertionUtils { + public static boolean isAuthorizedToEditAssertionFromAssertee( + final QueryContext context, final Urn asserteeUrn) { + final DisjunctivePrivilegeGroup orPrivilegeGroups = + new DisjunctivePrivilegeGroup( + ImmutableList.of( + AuthorizationUtils.ALL_PRIVILEGES_GROUP, + new ConjunctivePrivilegeGroup( + ImmutableList.of(PoliciesConfig.EDIT_ENTITY_ASSERTIONS_PRIVILEGE.getType())))); + return AuthorizationUtils.isAuthorized( + context.getAuthorizer(), + context.getActorUrn(), + asserteeUrn.getEntityType(), + asserteeUrn.toString(), + orPrivilegeGroups); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/ReportAssertionResultResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/ReportAssertionResultResolver.java new file mode 100644 index 0000000000000..b720aa11a8bdc --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/ReportAssertionResultResolver.java @@ -0,0 +1,117 @@ +package com.linkedin.datahub.graphql.resolvers.assertion; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + +import com.linkedin.assertion.AssertionResult; +import com.linkedin.assertion.AssertionResultError; +import com.linkedin.assertion.AssertionResultErrorType; +import com.linkedin.assertion.AssertionResultType; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.StringMap; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.AssertionResultInput; +import com.linkedin.datahub.graphql.generated.StringMapEntryInput; +import com.linkedin.metadata.service.AssertionService; +import graphql.execution.DataFetcherExceptionHandler; +import graphql.execution.DataFetcherResult; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ReportAssertionResultResolver implements DataFetcher> { + + public static final String ERROR_MESSAGE_KEY = "message"; + private final AssertionService _assertionService; + + public ReportAssertionResultResolver(AssertionService assertionService) { + _assertionService = assertionService; + } + + /** + * This is called by the graphql engine to fetch the value. The {@link DataFetchingEnvironment} is + * a composite context object that tells you all you need to know about how to fetch a data value + * in graphql type terms. + * + * @param environment this is the data fetching environment which contains all the context you + * need to fetch a value + * @return a value of type T. May be wrapped in a {@link DataFetcherResult} + * @throws Exception to relieve the implementations from having to wrap checked exceptions. Any + * exception thrown from a {@code DataFetcher} will eventually be handled by the registered + * {@link DataFetcherExceptionHandler} and the related field will have a value of {@code null} + * in the result. + */ + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final Urn assertionUrn = UrnUtils.getUrn(environment.getArgument("urn")); + final AssertionResultInput input = + bindArgument(environment.getArgument("result"), AssertionResultInput.class); + + return CompletableFuture.supplyAsync( + () -> { + final Urn asserteeUrn = + _assertionService.getEntityUrnForAssertion( + context.getOperationContext(), assertionUrn); + if (asserteeUrn == null) { + throw new RuntimeException( + String.format( + "Failed to report Assertion Run Event. Assertion with urn %s does not exist or is not associated with any entity.", + assertionUrn)); + } + + // Check whether the current user is allowed to update the assertion. + if (AssertionUtils.isAuthorizedToEditAssertionFromAssertee(context, asserteeUrn)) { + AssertionResult assertionResult = mapAssertionResult(input); + _assertionService.addAssertionRunEvent( + context.getOperationContext(), + assertionUrn, + asserteeUrn, + input.getTimestampMillis() != null + ? input.getTimestampMillis() + : System.currentTimeMillis(), + assertionResult); + return true; + } + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + }); + } + + private static StringMap mapContextParameters(List input) { + + if (input == null || input.isEmpty()) { + return null; + } + StringMap entries = new StringMap(); + input.forEach(entry -> entries.put(entry.getKey(), entry.getValue())); + return entries; + } + + private AssertionResult mapAssertionResult(AssertionResultInput input) { + AssertionResult assertionResult = new AssertionResult(); + assertionResult.setType(AssertionResultType.valueOf(input.getType().toString())); + assertionResult.setExternalUrl(input.getExternalUrl(), SetMode.IGNORE_NULL); + if (assertionResult.getType() == AssertionResultType.ERROR && input.getError() != null) { + assertionResult.setError(mapAssertionResultError(input)); + } + if (input.getProperties() != null) { + assertionResult.setNativeResults(mapContextParameters(input.getProperties())); + } + return assertionResult; + } + + private static AssertionResultError mapAssertionResultError(AssertionResultInput input) { + AssertionResultError error = new AssertionResultError(); + error.setType(AssertionResultErrorType.valueOf(input.getError().getType().toString())); + StringMap errorProperties = new StringMap(); + errorProperties.put(ERROR_MESSAGE_KEY, input.getError().getMessage()); + error.setProperties(errorProperties); + return error; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/UpsertCustomAssertionResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/UpsertCustomAssertionResolver.java new file mode 100644 index 0000000000000..026f486e32c11 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/UpsertCustomAssertionResolver.java @@ -0,0 +1,108 @@ +package com.linkedin.datahub.graphql.resolvers.assertion; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static com.linkedin.metadata.Constants.*; + +import com.linkedin.assertion.CustomAssertionInfo; +import com.linkedin.common.DataPlatformInstance; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.Assertion; +import com.linkedin.datahub.graphql.generated.PlatformInput; +import com.linkedin.datahub.graphql.generated.UpsertCustomAssertionInput; +import com.linkedin.datahub.graphql.types.assertion.AssertionMapper; +import com.linkedin.metadata.key.DataPlatformKey; +import com.linkedin.metadata.service.AssertionService; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.metadata.utils.SchemaFieldUtils; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class UpsertCustomAssertionResolver implements DataFetcher> { + + private final AssertionService _assertionService; + + public UpsertCustomAssertionResolver(@Nonnull final AssertionService assertionService) { + _assertionService = Objects.requireNonNull(assertionService, "assertionService is required"); + } + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final String maybeAssertionUrn = environment.getArgument("urn"); + final UpsertCustomAssertionInput input = + bindArgument(environment.getArgument("input"), UpsertCustomAssertionInput.class); + + final Urn entityUrn = UrnUtils.getUrn(input.getEntityUrn()); + final Urn assertionUrn; + + if (maybeAssertionUrn == null) { + assertionUrn = _assertionService.generateAssertionUrn(); + } else { + assertionUrn = UrnUtils.getUrn(maybeAssertionUrn); + } + + return CompletableFuture.supplyAsync( + () -> { + // Check whether the current user is allowed to update the assertion. + if (AssertionUtils.isAuthorizedToEditAssertionFromAssertee(context, entityUrn)) { + _assertionService.upsertCustomAssertion( + context.getOperationContext(), + assertionUrn, + entityUrn, + input.getDescription(), + input.getExternalUrl(), + mapAssertionPlatform(input.getPlatform()), + createCustomAssertionInfo(input, entityUrn)); + + return AssertionMapper.map( + context, + _assertionService.getAssertionEntityResponse( + context.getOperationContext(), assertionUrn)); + } + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + }); + } + + @SneakyThrows + private DataPlatformInstance mapAssertionPlatform(PlatformInput platformInput) { + DataPlatformInstance platform = new DataPlatformInstance(); + if (platformInput.getUrn() != null) { + platform.setPlatform(Urn.createFromString(platformInput.getUrn())); + } else if (platformInput.getName() != null) { + platform.setPlatform( + EntityKeyUtils.convertEntityKeyToUrn( + new DataPlatformKey().setPlatformName(platformInput.getName()), + DATA_PLATFORM_ENTITY_NAME)); + } else { + throw new IllegalArgumentException( + "Failed to upsert Custom Assertion. Platform Name or Platform Urn must be specified."); + } + + return platform; + } + + private CustomAssertionInfo createCustomAssertionInfo( + UpsertCustomAssertionInput input, Urn entityUrn) { + CustomAssertionInfo customAssertionInfo = new CustomAssertionInfo(); + customAssertionInfo.setType(input.getType()); + customAssertionInfo.setEntity(entityUrn); + customAssertionInfo.setLogic(input.getLogic(), SetMode.IGNORE_NULL); + + if (input.getFieldPath() != null) { + customAssertionInfo.setField( + SchemaFieldUtils.generateSchemaFieldUrn(entityUrn.toString(), input.getFieldPath())); + } + return customAssertionInfo; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java index eaac2aedef03a..dc57ed3c673c1 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java @@ -70,7 +70,10 @@ public CompletableFuture get(DataFetchingEnvironment envi .withSearchFlags(flags -> flags.setFulltext(true)), Constants.ACCESS_TOKEN_ENTITY_NAME, "", - buildFilter(filters, Collections.emptyList()), + buildFilter( + filters, + Collections.emptyList(), + context.getOperationContext().getAspectRetriever()), sortCriterion, start, count); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java index 18ee5f595ce58..b54ca398aef98 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java @@ -74,7 +74,9 @@ public CompletableFuture get(DataFetchingEnvironment environmen ? BROWSE_PATH_V2_DELIMITER + String.join(BROWSE_PATH_V2_DELIMITER, input.getPath()) : ""; - final Filter inputFilter = ResolverUtils.buildFilter(null, input.getOrFilters()); + final Filter inputFilter = + ResolverUtils.buildFilter( + null, input.getOrFilters(), context.getOperationContext().getAspectRetriever()); BrowseResultV2 browseResults = _entityClient.browseV2( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java index c05009e146308..caa469003c22e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java @@ -185,6 +185,7 @@ public CompletableFuture get(final DataFetchingEnvironment environmen .setShowAccessManagement(_featureFlags.isShowAccessManagement()) .setNestedDomainsEnabled(_featureFlags.isNestedDomainsEnabled()) .setPlatformBrowseV2(_featureFlags.isPlatformBrowseV2()) + .setDataContractsEnabled(_featureFlags.isDataContractsEnabled()) .build(); appConfig.setFeatureFlags(featureFlagsConfig); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/datacontract/DataContractUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/datacontract/DataContractUtils.java new file mode 100644 index 0000000000000..3dd7cd9df6383 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/datacontract/DataContractUtils.java @@ -0,0 +1,32 @@ +package com.linkedin.datahub.graphql.resolvers.datacontract; + +import com.datahub.authorization.ConjunctivePrivilegeGroup; +import com.datahub.authorization.DisjunctivePrivilegeGroup; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.metadata.authorization.PoliciesConfig; +import javax.annotation.Nonnull; + +public class DataContractUtils { + + public static boolean canEditDataContract(@Nonnull QueryContext context, Urn entityUrn) { + final DisjunctivePrivilegeGroup orPrivilegeGroups = + new DisjunctivePrivilegeGroup( + ImmutableList.of( + AuthorizationUtils.ALL_PRIVILEGES_GROUP, + new ConjunctivePrivilegeGroup( + ImmutableList.of( + PoliciesConfig.EDIT_ENTITY_DATA_CONTRACT_PRIVILEGE.getType())))); + + return AuthorizationUtils.isAuthorized( + context.getAuthorizer(), + context.getActorUrn(), + entityUrn.getEntityType(), + entityUrn.toString(), + orPrivilegeGroups); + } + + private DataContractUtils() {} +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/datacontract/EntityDataContractResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/datacontract/EntityDataContractResolver.java new file mode 100644 index 0000000000000..338e7a0015b5a --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/datacontract/EntityDataContractResolver.java @@ -0,0 +1,96 @@ +package com.linkedin.datahub.graphql.resolvers.datacontract; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.EntityRelationship; +import com.linkedin.common.EntityRelationships; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DataContract; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.types.datacontract.DataContractMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.graph.GraphClient; +import com.linkedin.metadata.query.filter.RelationshipDirection; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class EntityDataContractResolver implements DataFetcher> { + static final String CONTRACT_FOR_RELATIONSHIP = "ContractFor"; + + private final EntityClient _entityClient; + private final GraphClient _graphClient; + + public EntityDataContractResolver( + final EntityClient entityClient, final GraphClient graphClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + _graphClient = Objects.requireNonNull(graphClient, "graphClient must not be null"); + } + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) { + return CompletableFuture.supplyAsync( + () -> { + final QueryContext context = environment.getContext(); + final String entityUrn = ((Entity) environment.getSource()).getUrn(); + + try { + // Step 1: Fetch the contract associated with the dataset. + final EntityRelationships relationships = + _graphClient.getRelatedEntities( + entityUrn, + ImmutableList.of(CONTRACT_FOR_RELATIONSHIP), + RelationshipDirection.INCOMING, + 0, + 1, + context.getActorUrn()); + + // If we found multiple contracts for same entity, we have an invalid system state. Log + // a warning. + if (relationships.getTotal() > 1) { + // Someone created 2 contracts for the same entity. Currently, we do not handle this + // in the UI. + log.warn( + String.format( + "Unexpectedly found multiple contracts (%s) for entity with urn %s! This may lead to inconsistent behavior.", + relationships.getRelationships(), entityUrn)); + } + + final List contractUrns = + relationships.getRelationships().stream() + .map(EntityRelationship::getEntity) + .collect(Collectors.toList()); + + if (!contractUrns.isEmpty()) { + final Urn contractUrn = contractUrns.get(0); + + // Step 2: Hydrate the contract entities based on the urns from step 1 + final EntityResponse entityResponse = + _entityClient.getV2( + context.getOperationContext(), + Constants.DATA_CONTRACT_ENTITY_NAME, + contractUrn, + null); + + if (entityResponse != null) { + // Step 4: Package and return result + return DataContractMapper.mapContract(entityResponse); + } + } + // No contract found + return null; + } catch (URISyntaxException | RemoteInvocationException e) { + throw new RuntimeException("Failed to retrieve Data Contract from GMS", e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/datacontract/UpsertDataContractResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/datacontract/UpsertDataContractResolver.java new file mode 100644 index 0000000000000..955a4ed0ee6b2 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/datacontract/UpsertDataContractResolver.java @@ -0,0 +1,278 @@ +package com.linkedin.datahub.graphql.resolvers.datacontract; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.EntityRelationships; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datacontract.DataContractProperties; +import com.linkedin.datacontract.DataContractState; +import com.linkedin.datacontract.DataContractStatus; +import com.linkedin.datacontract.DataQualityContract; +import com.linkedin.datacontract.DataQualityContractArray; +import com.linkedin.datacontract.FreshnessContract; +import com.linkedin.datacontract.FreshnessContractArray; +import com.linkedin.datacontract.SchemaContract; +import com.linkedin.datacontract.SchemaContractArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLException; +import com.linkedin.datahub.graphql.generated.DataContract; +import com.linkedin.datahub.graphql.generated.DataQualityContractInput; +import com.linkedin.datahub.graphql.generated.FreshnessContractInput; +import com.linkedin.datahub.graphql.generated.SchemaContractInput; +import com.linkedin.datahub.graphql.generated.UpsertDataContractInput; +import com.linkedin.datahub.graphql.types.datacontract.DataContractMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.graph.GraphClient; +import com.linkedin.metadata.key.DataContractKey; +import com.linkedin.metadata.query.filter.RelationshipDirection; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class UpsertDataContractResolver implements DataFetcher> { + + private static final DataContractState DEFAULT_CONTRACT_STATE = DataContractState.ACTIVE; + private static final String CONTRACT_RELATIONSHIP_TYPE = "ContractFor"; + private final EntityClient _entityClient; + private final GraphClient _graphClient; + + public UpsertDataContractResolver( + final EntityClient entityClient, final GraphClient graphClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient cannot be null"); + _graphClient = Objects.requireNonNull(graphClient, "graphClient cannot be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) + throws Exception { + final QueryContext context = environment.getContext(); + final UpsertDataContractInput input = + bindArgument(environment.getArgument("input"), UpsertDataContractInput.class); + final Urn entityUrn = UrnUtils.getUrn(input.getEntityUrn()); + return CompletableFuture.supplyAsync( + () -> { + if (DataContractUtils.canEditDataContract(context, entityUrn)) { + + // Verify that the provided contract, dataset, assertions all exist as valid entities. + validateInput(entityUrn, input, context); + + // First determine if there is an existing data contract + final Urn maybeExistingContractUrn = + getEntityContractUrn(entityUrn, context.getAuthentication()); + + final DataContractProperties newProperties = mapInputToProperties(entityUrn, input); + final DataContractStatus newStatus = mapInputToStatus(input); + + final Urn urn = + maybeExistingContractUrn != null + ? maybeExistingContractUrn + : EntityKeyUtils.convertEntityKeyToUrn( + new DataContractKey() + .setId( + input.getId() != null + ? input.getId() + : UUID.randomUUID().toString()), + Constants.DATA_CONTRACT_ENTITY_NAME); + + final MetadataChangeProposal propertiesProposal = + buildMetadataChangeProposalWithUrn( + urn, Constants.DATA_CONTRACT_PROPERTIES_ASPECT_NAME, newProperties); + + final MetadataChangeProposal statusProposal = + buildMetadataChangeProposalWithUrn( + urn, Constants.DATA_CONTRACT_STATUS_ASPECT_NAME, newStatus); + + try { + _entityClient.batchIngestProposals( + context.getOperationContext(), + ImmutableList.of(propertiesProposal, statusProposal), + false); + + // Hydrate the contract entities based on the urns from step 1 + final EntityResponse entityResponse = + _entityClient.getV2( + context.getOperationContext(), + Constants.DATA_CONTRACT_ENTITY_NAME, + urn, + null); + + // Package and return result + return DataContractMapper.mapContract(entityResponse); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input.toString()), e); + } + } + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + }); + } + + private void validateInput( + @Nonnull final Urn entityUrn, + @Nonnull final UpsertDataContractInput input, + @Nonnull final QueryContext context) { + try { + + // Validate the target entity exists + if (!_entityClient.exists(context.getOperationContext(), entityUrn)) { + throw new DataHubGraphQLException( + String.format("Provided entity with urn %s does not exist!", entityUrn), + DataHubGraphQLErrorCode.BAD_REQUEST); + } + + // Verify Freshness assertions + if (input.getFreshness() != null) { + final List freshnessInputs = input.getFreshness(); + for (FreshnessContractInput freshnessInput : freshnessInputs) { + final Urn assertionUrn = UrnUtils.getUrn(freshnessInput.getAssertionUrn()); + if (!_entityClient.exists(context.getOperationContext(), assertionUrn)) { + throw new DataHubGraphQLException( + String.format("Provided assertion with urn %s does not exist!", assertionUrn), + DataHubGraphQLErrorCode.BAD_REQUEST); + } + } + } + + // Verify Schema assertions + if (input.getSchema() != null) { + final List schemaInputs = input.getSchema(); + for (SchemaContractInput schemaInput : schemaInputs) { + final Urn assertionUrn = UrnUtils.getUrn(schemaInput.getAssertionUrn()); + if (!_entityClient.exists(context.getOperationContext(), assertionUrn)) { + throw new DataHubGraphQLException( + String.format("Provided assertion with urn %s does not exist!", assertionUrn), + DataHubGraphQLErrorCode.BAD_REQUEST); + } + } + } + + // Verify DQ assertions + if (input.getDataQuality() != null) { + final List dqInputs = input.getDataQuality(); + for (DataQualityContractInput dqInput : dqInputs) { + final Urn assertionUrn = UrnUtils.getUrn(dqInput.getAssertionUrn()); + if (!_entityClient.exists(context.getOperationContext(), assertionUrn)) { + throw new DataHubGraphQLException( + String.format("Provided assertion with urn %s does not exist!", assertionUrn), + DataHubGraphQLErrorCode.BAD_REQUEST); + } + } + } + } catch (Exception e) { + if (e instanceof DataHubGraphQLException) { + throw (DataHubGraphQLException) e; + } else { + log.error( + "Failed to validate inputs provided when upserting data contract! Failing the create.", + e); + throw new DataHubGraphQLException( + "Failed to verify inputs. An unknown error occurred!", + DataHubGraphQLErrorCode.SERVER_ERROR); + } + } + } + + @Nullable + private Urn getEntityContractUrn(@Nonnull Urn entityUrn, @Nonnull Authentication authentication) { + EntityRelationships relationships = + _graphClient.getRelatedEntities( + entityUrn.toString(), + ImmutableList.of(CONTRACT_RELATIONSHIP_TYPE), + RelationshipDirection.INCOMING, + 0, + 1, + authentication.getActor().toUrnStr()); + + if (relationships.getTotal() > 1) { + // Bad state - There are multiple contracts for a single entity! Cannot update. + log.warn( + String.format( + "Unexpectedly found multiple contracts (%s) for entity with urn %s! This may lead to inconsistent behavior.", + relationships.getRelationships(), entityUrn)); + } + + if (relationships.getRelationships().size() == 1) { + return relationships.getRelationships().get(0).getEntity(); + } + // No Contract Found + return null; + } + + private DataContractProperties mapInputToProperties( + @Nonnull final Urn entityUrn, @Nonnull final UpsertDataContractInput input) { + final DataContractProperties result = new DataContractProperties(); + result.setEntity(entityUrn); + + // Construct the dataset contract. + if (input.getFreshness() != null) { + result.setFreshness( + new FreshnessContractArray( + input.getFreshness().stream() + .map(this::mapFreshnessInput) + .collect(Collectors.toList()))); + } + + if (input.getSchema() != null) { + result.setSchema( + new SchemaContractArray( + input.getSchema().stream().map(this::mapSchemaInput).collect(Collectors.toList()))); + } + + if (input.getDataQuality() != null) { + result.setDataQuality( + new DataQualityContractArray( + input.getDataQuality().stream() + .map(this::mapDataQualityInput) + .collect(Collectors.toList()))); + } + + return result; + } + + private DataContractStatus mapInputToStatus(@Nonnull final UpsertDataContractInput input) { + final DataContractStatus result = new DataContractStatus(); + if (input.getState() != null) { + result.setState(DataContractState.valueOf(input.getState().toString())); + } else { + result.setState(DEFAULT_CONTRACT_STATE); + } + return result; + } + + private FreshnessContract mapFreshnessInput(@Nonnull final FreshnessContractInput input) { + final FreshnessContract result = new FreshnessContract(); + result.setAssertion(UrnUtils.getUrn(input.getAssertionUrn())); + return result; + } + + private SchemaContract mapSchemaInput(@Nonnull final SchemaContractInput input) { + final SchemaContract result = new SchemaContract(); + result.setAssertion(UrnUtils.getUrn(input.getAssertionUrn())); + return result; + } + + private DataQualityContract mapDataQualityInput(@Nonnull final DataQualityContractInput input) { + final DataQualityContract result = new DataQualityContract(); + result.setAssertion(UrnUtils.getUrn(input.getAssertionUrn())); + return result; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java index 041de9f58db23..320d89cdec164 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java @@ -132,7 +132,10 @@ public CompletableFuture get(DataFetchingEnvironment environment) // add urns from the aspect to our filters final Filter baseFilter = - ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); + ResolverUtils.buildFilter( + input.getFilters(), + input.getOrFilters(), + context.getOperationContext().getAspectRetriever()); final Filter finalFilter = buildFilterWithUrns(new HashSet<>(assetUrns), baseFilter); final SearchFlags searchFlags; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java index 9ca5de86034da..75796f637525e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java @@ -78,7 +78,9 @@ public CompletableFuture get(final DataFetchingEnvironment enviro .getFilters() .forEach( filter -> { - criteria.add(criterionFromFilter(filter, true)); + criteria.add( + criterionFromFilter( + filter, true, context.getOperationContext().getAspectRetriever())); }); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java index b9d74f8af660e..3cf4d9175d45b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java @@ -33,7 +33,9 @@ public CompletableFuture get(final DataFetchingEnvironment environment) final CreateDynamicFormAssignmentInput input = bindArgument(environment.getArgument("input"), CreateDynamicFormAssignmentInput.class); final Urn formUrn = UrnUtils.getUrn(input.getFormUrn()); - final DynamicFormAssignment formAssignment = FormUtils.mapDynamicFormAssignment(input); + final DynamicFormAssignment formAssignment = + FormUtils.mapDynamicFormAssignment( + input, context.getOperationContext().getAspectRetriever()); return GraphQLConcurrencyUtils.supplyAsync( () -> { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateFormResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateFormResolver.java new file mode 100644 index 0000000000000..e9962464059e6 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateFormResolver.java @@ -0,0 +1,83 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.CreateFormInput; +import com.linkedin.datahub.graphql.generated.CreatePromptInput; +import com.linkedin.datahub.graphql.generated.Form; +import com.linkedin.datahub.graphql.generated.FormPromptType; +import com.linkedin.datahub.graphql.resolvers.mutate.util.FormUtils; +import com.linkedin.datahub.graphql.types.form.FormMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.form.FormInfo; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.service.FormService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class CreateFormResolver implements DataFetcher> { + + private final EntityClient _entityClient; + private final FormService _formService; + + public CreateFormResolver( + @Nonnull final EntityClient entityClient, @Nonnull final FormService formService) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + _formService = Objects.requireNonNull(formService, "formService must not be null"); + } + + @Override + public CompletableFuture
get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + + final CreateFormInput input = + bindArgument(environment.getArgument("input"), CreateFormInput.class); + final FormInfo formInfo = FormUtils.mapFormInfo(input); + + return CompletableFuture.supplyAsync( + () -> { + try { + if (!AuthorizationUtils.canManageForms(context)) { + throw new AuthorizationException("Unable to create form. Please contact your admin."); + } + validatePrompts(input.getPrompts()); + + Urn formUrn = + _formService.createForm(context.getOperationContext(), formInfo, input.getId()); + EntityResponse response = + _entityClient.getV2( + context.getOperationContext(), Constants.FORM_ENTITY_NAME, formUrn, null); + return FormMapper.map(context, response); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } + + private void validatePrompts(@Nullable List prompts) { + if (prompts == null) { + return; + } + prompts.forEach( + prompt -> { + if (prompt.getType().equals(FormPromptType.STRUCTURED_PROPERTY) + || prompt.getType().equals(FormPromptType.FIELDS_STRUCTURED_PROPERTY)) { + if (prompt.getStructuredPropertyParams() == null) { + throw new IllegalArgumentException( + "Provided prompt with type STRUCTURED_PROPERTY or FIELDS_STRUCTURED_PROPERTY and no structured property params"); + } + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/DeleteFormResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/DeleteFormResolver.java new file mode 100644 index 0000000000000..eec6816042a40 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/DeleteFormResolver.java @@ -0,0 +1,65 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.DeleteFormInput; +import com.linkedin.entity.client.EntityClient; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DeleteFormResolver implements DataFetcher> { + + private final EntityClient _entityClient; + + public DeleteFormResolver(@Nonnull final EntityClient entityClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) + throws Exception { + final QueryContext context = environment.getContext(); + + final DeleteFormInput input = + bindArgument(environment.getArgument("input"), DeleteFormInput.class); + final Urn formUrn = UrnUtils.getUrn(input.getUrn()); + + return CompletableFuture.supplyAsync( + () -> { + try { + if (!AuthorizationUtils.canManageForms(context)) { + throw new AuthorizationException("Unable to delete form. Please contact your admin."); + } + _entityClient.deleteEntity(context.getOperationContext(), formUrn); + // Asynchronously Delete all references to the entity (to return quickly) + CompletableFuture.runAsync( + () -> { + try { + _entityClient.deleteEntityReferences(context.getOperationContext(), formUrn); + } catch (Exception e) { + log.error( + String.format( + "Caught exception while attempting to clear all entity references for Form with urn %s", + formUrn), + e); + } + }); + + return true; + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/UpdateFormResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/UpdateFormResolver.java new file mode 100644 index 0000000000000..8b4d1debcd4db --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/UpdateFormResolver.java @@ -0,0 +1,98 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.Form; +import com.linkedin.datahub.graphql.generated.UpdateFormInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.FormUtils; +import com.linkedin.datahub.graphql.types.form.FormMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.form.FormType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.patch.builder.FormInfoPatchBuilder; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; + +public class UpdateFormResolver implements DataFetcher> { + + private final EntityClient _entityClient; + + public UpdateFormResolver(@Nonnull final EntityClient entityClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + + final UpdateFormInput input = + bindArgument(environment.getArgument("input"), UpdateFormInput.class); + final Urn formUrn = UrnUtils.getUrn(input.getUrn()); + + return CompletableFuture.supplyAsync( + () -> { + try { + if (!AuthorizationUtils.canManageForms(context)) { + throw new AuthorizationException("Unable to update form. Please contact your admin."); + } + if (!_entityClient.exists(context.getOperationContext(), formUrn)) { + throw new IllegalArgumentException( + String.format("Form with urn %s does not exist", formUrn)); + } + + FormInfoPatchBuilder patchBuilder = new FormInfoPatchBuilder().urn(formUrn); + if (input.getName() != null) { + patchBuilder.setName(input.getName()); + } + if (input.getDescription() != null) { + patchBuilder.setDescription(input.getDescription()); + } + if (input.getType() != null) { + patchBuilder.setType(FormType.valueOf(input.getType().toString())); + } + if (input.getPromptsToAdd() != null) { + patchBuilder.addPrompts(FormUtils.mapPromptsToAdd(input.getPromptsToAdd())); + } + if (input.getPromptsToRemove() != null) { + patchBuilder.removePrompts(input.getPromptsToRemove()); + } + if (input.getActors() != null) { + if (input.getActors().getOwners() != null) { + patchBuilder.setOwnershipForm(input.getActors().getOwners()); + } + if (input.getActors().getUsersToAdd() != null) { + input.getActors().getUsersToAdd().forEach(patchBuilder::addAssignedUser); + } + if (input.getActors().getUsersToRemove() != null) { + input.getActors().getUsersToRemove().forEach(patchBuilder::removeAssignedUser); + } + if (input.getActors().getGroupsToAdd() != null) { + input.getActors().getGroupsToAdd().forEach(patchBuilder::addAssignedGroup); + } + if (input.getActors().getGroupsToRemove() != null) { + input.getActors().getGroupsToRemove().forEach(patchBuilder::removeAssignedGroup); + } + } + _entityClient.ingestProposal( + context.getOperationContext(), patchBuilder.build(), false); + + EntityResponse response = + _entityClient.getV2( + context.getOperationContext(), Constants.FORM_ENTITY_NAME, formUrn, null); + return FormMapper.map(context, response); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java index 8ead47aa65ceb..1a2806224e4a9 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java @@ -68,7 +68,10 @@ public CompletableFuture get( .withSearchFlags(flags -> flags.setFulltext(true)), Constants.INGESTION_SOURCE_ENTITY_NAME, query, - buildFilter(filters, Collections.emptyList()), + buildFilter( + filters, + Collections.emptyList(), + context.getOperationContext().getAspectRetriever()), null, start, count); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java index 1839fd3cc5705..8fc26e3cec9d0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java @@ -11,6 +11,7 @@ import com.linkedin.datahub.graphql.generated.TimeSeriesAspect; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.EnvelopedAspect; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; @@ -120,7 +121,7 @@ public CompletableFuture> get(DataFetchingEnvironment env maybeStartTimeMillis, maybeEndTimeMillis, maybeLimit, - buildFilters(maybeFilters), + buildFilters(maybeFilters, context.getOperationContext().getAspectRetriever()), maybeSort); // Step 2: Bind profiles into GraphQL strong types. @@ -135,7 +136,8 @@ public CompletableFuture> get(DataFetchingEnvironment env "get"); } - private Filter buildFilters(@Nullable FilterInput maybeFilters) { + private Filter buildFilters( + @Nullable FilterInput maybeFilters, @Nullable AspectRetriever aspectRetriever) { if (maybeFilters == null) { return null; } @@ -146,7 +148,7 @@ private Filter buildFilters(@Nullable FilterInput maybeFilters) { .setAnd( new CriterionArray( maybeFilters.getAnd().stream() - .map(filter -> criterionFromFilter(filter, true)) + .map(filter -> criterionFromFilter(filter, true, aspectRetriever)) .collect(Collectors.toList()))))); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java index 9a06682c87f78..17718f39c1238 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java @@ -1,11 +1,24 @@ package com.linkedin.datahub.graphql.resolvers.mutate.util; +import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.generated.CreateDynamicFormAssignmentInput; +import com.linkedin.datahub.graphql.generated.CreateFormInput; +import com.linkedin.datahub.graphql.generated.CreatePromptInput; +import com.linkedin.datahub.graphql.generated.FormActorAssignmentInput; +import com.linkedin.datahub.graphql.generated.StructuredPropertyParamsInput; import com.linkedin.datahub.graphql.generated.SubmitFormPromptInput; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.form.DynamicFormAssignment; +import com.linkedin.form.FormActorAssignment; import com.linkedin.form.FormInfo; +import com.linkedin.form.FormPrompt; +import com.linkedin.form.FormPromptArray; +import com.linkedin.form.FormPromptType; +import com.linkedin.form.FormType; +import com.linkedin.form.StructuredPropertyParams; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; @@ -13,8 +26,12 @@ import com.linkedin.metadata.query.filter.CriterionArray; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.structured.PrimitivePropertyValueArray; +import java.util.List; import java.util.Objects; +import java.util.UUID; +import java.util.stream.Collectors; import javax.annotation.Nonnull; +import javax.annotation.Nullable; public class FormUtils { @@ -44,13 +61,16 @@ public static PrimitivePropertyValueArray getStructuredPropertyValuesFromInput( /** Map a GraphQL CreateDynamicFormAssignmentInput to the GMS DynamicFormAssignment aspect */ @Nonnull public static DynamicFormAssignment mapDynamicFormAssignment( - @Nonnull final CreateDynamicFormAssignmentInput input) { + @Nonnull final CreateDynamicFormAssignmentInput input, + @Nullable AspectRetriever aspectRetriever) { Objects.requireNonNull(input, "input must not be null"); final DynamicFormAssignment result = new DynamicFormAssignment(); final Filter filter = new Filter() - .setOr(ResolverUtils.buildConjunctiveCriterionArrayWithOr(input.getOrFilters())); + .setOr( + ResolverUtils.buildConjunctiveCriterionArrayWithOr( + input.getOrFilters(), aspectRetriever)); result.setFilter(filter); return result; } @@ -94,4 +114,103 @@ private static boolean isActorExplicitlyAssigned( || (formInfo.getActors().getGroups() != null && formInfo.getActors().getGroups().stream().anyMatch(group -> group.equals(actorUrn))); } + + @Nonnull + public static FormInfo mapFormInfo(@Nonnull final CreateFormInput input) { + Objects.requireNonNull(input, "input must not be null"); + + final FormInfo result = new FormInfo(); + result.setName(input.getName()); + if (input.getDescription() != null) { + result.setDescription(input.getDescription()); + } + if (input.getType() != null) { + result.setType(FormType.valueOf(input.getType().toString())); + } + if (input.getPrompts() != null) { + result.setPrompts(mapPrompts(input.getPrompts())); + } + if (input.getActors() != null) { + result.setActors(mapFormActorAssignment(input.getActors())); + } + + return result; + } + + @Nonnull + public static FormPromptArray mapPrompts(@Nonnull final List promptInputs) { + Objects.requireNonNull(promptInputs, "promptInputs must not be null"); + + final FormPromptArray result = new FormPromptArray(); + promptInputs.forEach( + promptInput -> { + result.add(mapPrompt(promptInput)); + }); + return result; + } + + @Nonnull + public static FormPrompt mapPrompt(@Nonnull final CreatePromptInput promptInput) { + Objects.requireNonNull(promptInput, "promptInput must not be null"); + + final FormPrompt result = new FormPrompt(); + String promptId = + promptInput.getId() != null ? promptInput.getId() : UUID.randomUUID().toString(); + result.setId(promptId); + result.setTitle(promptInput.getTitle()); + if (promptInput.getDescription() != null) { + result.setDescription(promptInput.getDescription()); + } + if (promptInput.getType() != null) { + result.setType(FormPromptType.valueOf(promptInput.getType().toString())); + } + if (promptInput.getStructuredPropertyParams() != null) { + result.setStructuredPropertyParams( + mapStructuredPropertyParams(promptInput.getStructuredPropertyParams())); + } + if (promptInput.getRequired() != null) { + result.setRequired(promptInput.getRequired()); + } + + return result; + } + + @Nonnull + public static StructuredPropertyParams mapStructuredPropertyParams( + @Nonnull final StructuredPropertyParamsInput paramsInput) { + Objects.requireNonNull(paramsInput, "paramsInput must not be null"); + + final StructuredPropertyParams result = new StructuredPropertyParams(); + result.setUrn(UrnUtils.getUrn(paramsInput.getUrn())); + return result; + } + + @Nonnull + public static FormActorAssignment mapFormActorAssignment( + @Nonnull final FormActorAssignmentInput input) { + Objects.requireNonNull(input, "input must not be null"); + + final FormActorAssignment result = new FormActorAssignment(); + if (input.getOwners() != null) { + result.setOwners(input.getOwners()); + } + if (input.getUsers() != null) { + UrnArray userUrns = new UrnArray(); + input.getUsers().forEach(user -> userUrns.add(UrnUtils.getUrn(user))); + result.setUsers(userUrns); + } + if (input.getGroups() != null) { + UrnArray groupUrns = new UrnArray(); + input.getGroups().forEach(group -> groupUrns.add(UrnUtils.getUrn(group))); + result.setUsers(groupUrns); + } + + return result; + } + + @Nonnull + public static List mapPromptsToAdd( + @Nonnull final List promptsToAdd) { + return promptsToAdd.stream().map(FormUtils::mapPrompt).collect(Collectors.toList()); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ownership/ListOwnershipTypesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ownership/ListOwnershipTypesResolver.java index 0f0bb299eda83..9f6951e44dd73 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ownership/ListOwnershipTypesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ownership/ListOwnershipTypesResolver.java @@ -63,7 +63,10 @@ public CompletableFuture get(DataFetchingEnvironment e context.getOperationContext().withSearchFlags(flags -> flags.setFulltext(true)), Constants.OWNERSHIP_TYPE_ENTITY_NAME, query, - buildFilter(filters, Collections.emptyList()), + buildFilter( + filters, + Collections.emptyList(), + context.getOperationContext().getAspectRetriever()), DEFAULT_SORT_CRITERION, start, count); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java index 4120401e0150f..ce11451aa1913 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java @@ -59,7 +59,11 @@ public CompletableFuture get(final DataFetchingEnvironment e log.debug( "User {} listing policies with filters {}", context.getActorUrn(), filters.toString()); - final Filter filter = ResolverUtils.buildFilter(facetFilters, Collections.emptyList()); + final Filter filter = + ResolverUtils.buildFilter( + facetFilters, + Collections.emptyList(), + context.getOperationContext().getAspectRetriever()); return _policyFetcher .fetchPolicies(context.getOperationContext(), start, query, count, filter) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolver.java index b0d84942b1257..95be3a68e895c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolver.java @@ -15,6 +15,7 @@ import com.linkedin.datahub.graphql.generated.ListQueriesResult; import com.linkedin.datahub.graphql.generated.QueryEntity; import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; @@ -72,7 +73,7 @@ public CompletableFuture get(final DataFetchingEnvironment en flags -> flags.setFulltext(true).setSkipHighlighting(true)), QUERY_ENTITY_NAME, query, - buildFilters(input), + buildFilters(input, context.getOperationContext().getAspectRetriever()), sortCriterion, start, count); @@ -109,7 +110,8 @@ private List mapUnresolvedQueries(final List queryUrns) { } @Nullable - private Filter buildFilters(@Nonnull final ListQueriesInput input) { + private Filter buildFilters( + @Nonnull final ListQueriesInput input, @Nullable AspectRetriever aspectRetriever) { final AndFilterInput criteria = new AndFilterInput(); List andConditions = new ArrayList<>(); @@ -136,6 +138,6 @@ private Filter buildFilters(@Nonnull final ListQueriesInput input) { } criteria.setAnd(andConditions); - return buildFilter(Collections.emptyList(), ImmutableList.of(criteria)); + return buildFilter(Collections.emptyList(), ImmutableList.of(criteria), aspectRetriever); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java index 202c78a62c9ae..0181877864390 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java @@ -26,6 +26,7 @@ import com.linkedin.metadata.service.ViewService; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; +import io.datahubproject.metadata.context.OperationContext; import io.opentelemetry.extension.annotations.WithSpan; import java.net.URISyntaxException; import java.util.Collections; @@ -33,6 +34,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -62,7 +64,7 @@ public CompletableFuture get(DataFetchingEnvironment List modules = _recommendationsService.listRecommendations( context.getOperationContext(), - mapRequestContext(input.getRequestContext()), + mapRequestContext(context.getOperationContext(), input.getRequestContext()), viewFilter(context.getOperationContext(), _viewService, input.getViewUrn()), input.getLimit()); return ListRecommendationsResult.builder() @@ -83,7 +85,7 @@ public CompletableFuture get(DataFetchingEnvironment } private com.linkedin.metadata.recommendation.RecommendationRequestContext mapRequestContext( - RecommendationRequestContext requestContext) { + @Nonnull OperationContext opContext, RecommendationRequestContext requestContext) { com.linkedin.metadata.recommendation.ScenarioType mappedScenarioType; try { mappedScenarioType = @@ -103,7 +105,9 @@ private com.linkedin.metadata.recommendation.RecommendationRequestContext mapReq searchRequestContext.setFilters( new CriterionArray( requestContext.getSearchRequestContext().getFilters().stream() - .map(facetField -> criterionFromFilter(facetField)) + .map( + facetField -> + criterionFromFilter(facetField, opContext.getAspectRetriever())) .collect(Collectors.toList()))); } mappedRequestContext.setSearchRequestContext(searchRequestContext); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java index 10a09b95bfd6e..04a72b14eeb02 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java @@ -63,7 +63,9 @@ public CompletableFuture get(DataFetchingEnvironment environme UrnUtils.getUrn(input.getViewUrn())) : null; - final Filter inputFilter = ResolverUtils.buildFilter(null, input.getOrFilters()); + final Filter inputFilter = + ResolverUtils.buildFilter( + null, input.getOrFilters(), context.getOperationContext().getAspectRetriever()); final SearchFlags searchFlags = mapInputFlags(context, input.getSearchFlags()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java index 13861c94ba336..79792940ef27f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java @@ -3,6 +3,7 @@ import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; import static org.apache.commons.lang3.StringUtils.isBlank; +import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils; import com.linkedin.datahub.graphql.exception.ValidationException; import com.linkedin.datahub.graphql.generated.AutoCompleteInput; @@ -39,6 +40,7 @@ public AutoCompleteResolver(@Nonnull final List> sear @Override public CompletableFuture get(DataFetchingEnvironment environment) { + final QueryContext context = environment.getContext(); final AutoCompleteInput input = bindArgument(environment.getArgument("input"), AutoCompleteInput.class); @@ -49,7 +51,11 @@ public CompletableFuture get(DataFetchingEnvironment enviro throw new ValidationException("'query' parameter can not be null or empty"); } - final Filter filter = ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); + final Filter filter = + ResolverUtils.buildFilter( + input.getFilters(), + input.getOrFilters(), + context.getOperationContext().getRetrieverContext().orElseThrow().getAspectRetriever()); final int limit = input.getLimit() != null ? input.getLimit() : DEFAULT_LIMIT; return GraphQLConcurrencyUtils.supplyAsync( () -> { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java index c849e3ad3f68c..5b5888b89b241 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.resolvers.search; +import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils; import com.linkedin.datahub.graphql.generated.AutoCompleteMultipleInput; import com.linkedin.datahub.graphql.generated.AutoCompleteMultipleResults; @@ -33,6 +34,7 @@ public static CompletableFuture batchGetAutocomplet DataFetchingEnvironment environment, @Nullable DataHubViewInfo view) { final int limit = input.getLimit() != null ? input.getLimit() : DEFAULT_LIMIT; + final QueryContext context = environment.getContext(); final List> autoCompletesFuture = entities.stream() @@ -41,7 +43,10 @@ public static CompletableFuture batchGetAutocomplet GraphQLConcurrencyUtils.supplyAsync( () -> { final Filter filter = - ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); + ResolverUtils.buildFilter( + input.getFilters(), + input.getOrFilters(), + context.getOperationContext().getAspectRetriever()); final Filter finalFilter = view != null ? SearchUtils.combineFilters( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java index 77eef1b9a25c6..8b8b93353bc6e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java @@ -72,7 +72,9 @@ public CompletableFuture get(DataFetchingEnvironment environment) UrnUtils.getUrn(input.getViewUrn())) : null; - final Filter baseFilter = ResolverUtils.buildFilter(null, input.getOrFilters()); + final Filter baseFilter = + ResolverUtils.buildFilter( + null, input.getOrFilters(), context.getOperationContext().getAspectRetriever()); final SearchFlags searchFlags; com.linkedin.datahub.graphql.generated.SearchFlags inputFlags = input.getSearchFlags(); if (inputFlags != null) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java index addd217d68724..14b2d3b8f8420 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java @@ -129,7 +129,10 @@ public CompletableFuture get(DataFetchingEnvironment entityNames, sanitizedQuery, maxHops, - ResolverUtils.buildFilter(facetFilters, input.getOrFilters()), + ResolverUtils.buildFilter( + facetFilters, + input.getOrFilters(), + context.getOperationContext().getAspectRetriever()), null, scrollId, keepAlive, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java index a9da1c4055434..287e339ddee50 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java @@ -59,7 +59,10 @@ public CompletableFuture get(DataFetchingEnvironment environment) : null; final Filter baseFilter = - ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); + ResolverUtils.buildFilter( + input.getFilters(), + input.getOrFilters(), + context.getOperationContext().getAspectRetriever()); SearchFlags searchFlags = mapInputFlags(context, input.getSearchFlags()); SortCriterion sortCriterion = diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java index 238f2375ee207..f342d251acd72 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java @@ -138,7 +138,10 @@ public CompletableFuture get(DataFetchingEnvironment count); final Filter filter = - ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); + ResolverUtils.buildFilter( + input.getFilters(), + input.getOrFilters(), + context.getOperationContext().getAspectRetriever()); final SearchFlags searchFlags; com.linkedin.datahub.graphql.generated.SearchFlags inputFlags = input.getSearchFlags(); if (inputFlags != null) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java index 35586ea29571c..5fb2f8f14b293 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java @@ -85,7 +85,10 @@ public CompletableFuture get(DataFetchingEnvironment environment) context.getOperationContext().withSearchFlags(flags -> searchFlags), entityName, sanitizedQuery, - ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()), + ResolverUtils.buildFilter( + input.getFilters(), + input.getOrFilters(), + context.getOperationContext().getAspectRetriever()), null, start, count)); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java new file mode 100644 index 0000000000000..3be7ea505abbf --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java @@ -0,0 +1,136 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME; + +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.StringArray; +import com.linkedin.data.template.StringArrayMap; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.CreateStructuredPropertyInput; +import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertyMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.aspect.patch.builder.StructuredPropertyDefinitionPatchBuilder; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.structured.PrimitivePropertyValue; +import com.linkedin.structured.PropertyCardinality; +import com.linkedin.structured.PropertyValue; +import com.linkedin.structured.StructuredPropertyKey; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; + +public class CreateStructuredPropertyResolver + implements DataFetcher> { + + private final EntityClient _entityClient; + + public CreateStructuredPropertyResolver(@Nonnull final EntityClient entityClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) + throws Exception { + final QueryContext context = environment.getContext(); + + final CreateStructuredPropertyInput input = + bindArgument(environment.getArgument("input"), CreateStructuredPropertyInput.class); + + return CompletableFuture.supplyAsync( + () -> { + try { + if (!AuthorizationUtils.canManageStructuredProperties(context)) { + throw new AuthorizationException( + "Unable to create structured property. Please contact your admin."); + } + final StructuredPropertyKey key = new StructuredPropertyKey(); + final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString(); + key.setId(id); + final Urn propertyUrn = + EntityKeyUtils.convertEntityKeyToUrn(key, STRUCTURED_PROPERTY_ENTITY_NAME); + StructuredPropertyDefinitionPatchBuilder builder = + new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn); + + builder.setQualifiedName(input.getQualifiedName()); + builder.setValueType(input.getValueType()); + input.getEntityTypes().forEach(builder::addEntityType); + if (input.getDisplayName() != null) { + builder.setDisplayName(input.getDisplayName()); + } + if (input.getDescription() != null) { + builder.setDescription(input.getDescription()); + } + if (input.getImmutable() != null) { + builder.setImmutable(input.getImmutable()); + } + if (input.getTypeQualifier() != null) { + buildTypeQualifier(input, builder); + } + if (input.getAllowedValues() != null) { + buildAllowedValues(input, builder); + } + if (input.getCardinality() != null) { + builder.setCardinality( + PropertyCardinality.valueOf(input.getCardinality().toString())); + } + + MetadataChangeProposal mcp = builder.build(); + _entityClient.ingestProposal(context.getOperationContext(), mcp, false); + + EntityResponse response = + _entityClient.getV2( + context.getOperationContext(), + STRUCTURED_PROPERTY_ENTITY_NAME, + propertyUrn, + null); + return StructuredPropertyMapper.map(context, response); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } + + private void buildTypeQualifier( + @Nonnull final CreateStructuredPropertyInput input, + @Nonnull final StructuredPropertyDefinitionPatchBuilder builder) { + if (input.getTypeQualifier().getAllowedTypes() != null) { + final StringArrayMap typeQualifier = new StringArrayMap(); + StringArray allowedTypes = new StringArray(); + allowedTypes.addAll(input.getTypeQualifier().getAllowedTypes()); + typeQualifier.put("allowedTypes", allowedTypes); + builder.setTypeQualifier(typeQualifier); + } + } + + private void buildAllowedValues( + @Nonnull final CreateStructuredPropertyInput input, + @Nonnull final StructuredPropertyDefinitionPatchBuilder builder) { + input + .getAllowedValues() + .forEach( + allowedValueInput -> { + PropertyValue value = new PropertyValue(); + PrimitivePropertyValue primitiveValue = new PrimitivePropertyValue(); + if (allowedValueInput.getStringValue() != null) { + primitiveValue.setString(allowedValueInput.getStringValue()); + } + if (allowedValueInput.getNumberValue() != null) { + primitiveValue.setDouble(allowedValueInput.getNumberValue().doubleValue()); + } + value.setValue(primitiveValue); + value.setDescription(allowedValueInput.getDescription(), SetMode.IGNORE_NULL); + builder.addAllowedValue(value); + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java new file mode 100644 index 0000000000000..ea8c6dac36a4a --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java @@ -0,0 +1,103 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.RemoveStructuredPropertiesInput; +import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertiesMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.patch.builder.StructuredPropertiesPatchBuilder; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.structured.StructuredProperties; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; + +public class RemoveStructuredPropertiesResolver + implements DataFetcher< + CompletableFuture> { + + private final EntityClient _entityClient; + + public RemoveStructuredPropertiesResolver(@Nonnull final EntityClient entityClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + } + + @Override + public CompletableFuture get( + final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + + final RemoveStructuredPropertiesInput input = + bindArgument(environment.getArgument("input"), RemoveStructuredPropertiesInput.class); + final Urn assetUrn = UrnUtils.getUrn(input.getAssetUrn()); + + return CompletableFuture.supplyAsync( + () -> { + try { + // check authorization first + if (!AuthorizationUtils.canEditProperties(assetUrn, context)) { + throw new AuthorizationException( + String.format( + "Not authorized to update properties on the gives urn %s", assetUrn)); + } + + if (!_entityClient.exists(context.getOperationContext(), assetUrn)) { + throw new RuntimeException( + String.format("Asset with provided urn %s does not exist", assetUrn)); + } + + StructuredPropertiesPatchBuilder patchBuilder = + new StructuredPropertiesPatchBuilder().urn(assetUrn); + + input + .getStructuredPropertyUrns() + .forEach( + propertyUrn -> { + patchBuilder.removeProperty(UrnUtils.getUrn(propertyUrn)); + }); + + // ingest change proposal + final MetadataChangeProposal structuredPropertiesProposal = patchBuilder.build(); + + _entityClient.ingestProposal( + context.getOperationContext(), structuredPropertiesProposal, false); + + EntityResponse response = + _entityClient.getV2( + context.getOperationContext(), + assetUrn.getEntityType(), + assetUrn, + ImmutableSet.of(Constants.STRUCTURED_PROPERTIES_ASPECT_NAME)); + + if (response == null + || response.getAspects().get(Constants.STRUCTURED_PROPERTIES_ASPECT_NAME) == null) { + throw new RuntimeException( + String.format("Failed to fetch structured properties from entity %s", assetUrn)); + } + + StructuredProperties structuredProperties = + new StructuredProperties( + response + .getAspects() + .get(Constants.STRUCTURED_PROPERTIES_ASPECT_NAME) + .getValue() + .data()); + + return StructuredPropertiesMapper.map(context, structuredProperties); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolver.java new file mode 100644 index 0000000000000..2549f303bacd9 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolver.java @@ -0,0 +1,129 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.StringArray; +import com.linkedin.data.template.StringArrayMap; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.generated.UpdateStructuredPropertyInput; +import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertyMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.aspect.patch.builder.StructuredPropertyDefinitionPatchBuilder; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.structured.PrimitivePropertyValue; +import com.linkedin.structured.PropertyCardinality; +import com.linkedin.structured.PropertyValue; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; + +public class UpdateStructuredPropertyResolver + implements DataFetcher> { + + private final EntityClient _entityClient; + + public UpdateStructuredPropertyResolver(@Nonnull final EntityClient entityClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) + throws Exception { + final QueryContext context = environment.getContext(); + + final UpdateStructuredPropertyInput input = + bindArgument(environment.getArgument("input"), UpdateStructuredPropertyInput.class); + + return CompletableFuture.supplyAsync( + () -> { + try { + if (!AuthorizationUtils.canManageStructuredProperties(context)) { + throw new AuthorizationException( + "Unable to update structured property. Please contact your admin."); + } + final Urn propertyUrn = UrnUtils.getUrn(input.getUrn()); + StructuredPropertyDefinitionPatchBuilder builder = + new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn); + + if (input.getDisplayName() != null) { + builder.setDisplayName(input.getDisplayName()); + } + if (input.getDescription() != null) { + builder.setDescription(input.getDescription()); + } + if (input.getImmutable() != null) { + builder.setImmutable(input.getImmutable()); + } + if (input.getTypeQualifier() != null) { + buildTypeQualifier(input, builder); + } + if (input.getNewAllowedValues() != null) { + buildAllowedValues(input, builder); + } + if (input.getSetCardinalityAsMultiple() != null) { + builder.setCardinality(PropertyCardinality.MULTIPLE); + } + if (input.getNewEntityTypes() != null) { + input.getNewEntityTypes().forEach(builder::addEntityType); + } + + MetadataChangeProposal mcp = builder.build(); + _entityClient.ingestProposal(context.getOperationContext(), mcp, false); + + EntityResponse response = + _entityClient.getV2( + context.getOperationContext(), + STRUCTURED_PROPERTY_ENTITY_NAME, + propertyUrn, + null); + return StructuredPropertyMapper.map(context, response); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } + + private void buildTypeQualifier( + @Nonnull final UpdateStructuredPropertyInput input, + @Nonnull final StructuredPropertyDefinitionPatchBuilder builder) { + if (input.getTypeQualifier().getNewAllowedTypes() != null) { + final StringArrayMap typeQualifier = new StringArrayMap(); + StringArray allowedTypes = new StringArray(); + allowedTypes.addAll(input.getTypeQualifier().getNewAllowedTypes()); + typeQualifier.put("allowedTypes", allowedTypes); + builder.setTypeQualifier(typeQualifier); + } + } + + private void buildAllowedValues( + @Nonnull final UpdateStructuredPropertyInput input, + @Nonnull final StructuredPropertyDefinitionPatchBuilder builder) { + input + .getNewAllowedValues() + .forEach( + allowedValueInput -> { + PropertyValue value = new PropertyValue(); + PrimitivePropertyValue primitiveValue = new PrimitivePropertyValue(); + if (allowedValueInput.getStringValue() != null) { + primitiveValue.setString(allowedValueInput.getStringValue()); + } + if (allowedValueInput.getNumberValue() != null) { + primitiveValue.setDouble(allowedValueInput.getNumberValue().doubleValue()); + } + value.setValue(primitiveValue); + value.setDescription(allowedValueInput.getDescription(), SetMode.IGNORE_NULL); + builder.addAllowedValue(value); + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolver.java index 49c3467adb58b..7c3e433dd1ede 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolver.java @@ -49,7 +49,9 @@ public CompletableFuture get(final DataFetchingEnvironment environm DataHubViewType.valueOf(input.getViewType().toString()), input.getName(), input.getDescription(), - ViewUtils.mapDefinition(input.getDefinition()), + ViewUtils.mapDefinition( + input.getDefinition(), + context.getOperationContext().getAspectRetriever()), System.currentTimeMillis()); return createView(urn, input); } catch (Exception e) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java index 28b8fe50b70d6..952e55ca117f2 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java @@ -16,6 +16,7 @@ import com.linkedin.datahub.graphql.generated.ListViewsResult; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; @@ -30,6 +31,7 @@ import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; /** Resolver used for listing global DataHub Views. */ @@ -71,7 +73,7 @@ public CompletableFuture get(final DataFetchingEnvironment envi context.getOperationContext().withSearchFlags(flags -> flags.setFulltext(true)), Constants.DATAHUB_VIEW_ENTITY_NAME, query, - buildFilters(), + buildFilters(context.getOperationContext().getAspectRetriever()), DEFAULT_SORT_CRITERION, start, count); @@ -107,7 +109,7 @@ private List mapUnresolvedViews(final List entityUrns) { return results; } - private Filter buildFilters() { + private Filter buildFilters(@Nullable AspectRetriever aspectRetriever) { final AndFilterInput globalCriteria = new AndFilterInput(); List andConditions = new ArrayList<>(); andConditions.add( @@ -118,6 +120,6 @@ private Filter buildFilters() { false, FilterOperator.EQUAL)); globalCriteria.setAnd(andConditions); - return buildFilter(Collections.emptyList(), ImmutableList.of(globalCriteria)); + return buildFilter(Collections.emptyList(), ImmutableList.of(globalCriteria), aspectRetriever); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java index 218bedcd0beff..32eb0e46bb616 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java @@ -15,6 +15,7 @@ import com.linkedin.datahub.graphql.generated.ListViewsResult; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; @@ -74,7 +75,10 @@ public CompletableFuture get(final DataFetchingEnvironment envi context.getOperationContext().withSearchFlags(flags -> flags.setFulltext(true)), Constants.DATAHUB_VIEW_ENTITY_NAME, query, - buildFilters(viewType, context.getActorUrn()), + buildFilters( + viewType, + context.getActorUrn(), + context.getOperationContext().getAspectRetriever()), DEFAULT_SORT_CRITERION, start, count); @@ -110,7 +114,10 @@ private List mapUnresolvedViews(final List entityUrns) { return results; } - private Filter buildFilters(@Nullable final String viewType, final String creatorUrn) { + private Filter buildFilters( + @Nullable final String viewType, + final String creatorUrn, + @Nullable AspectRetriever aspectRetriever) { // And GLOBAL views for the authenticated actor. final AndFilterInput filterCriteria = new AndFilterInput(); final List andConditions = new ArrayList<>(); @@ -125,6 +132,6 @@ private Filter buildFilters(@Nullable final String viewType, final String creato filterCriteria.setAnd(andConditions); // Currently, there is no way to fetch the views belonging to another user. - return buildFilter(Collections.emptyList(), ImmutableList.of(filterCriteria)); + return buildFilter(Collections.emptyList(), ImmutableList.of(filterCriteria), aspectRetriever); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolver.java index 9ab5efe83105a..11ec1c5705bd3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolver.java @@ -48,7 +48,8 @@ public CompletableFuture get(final DataFetchingEnvironment environm urn, input.getName(), input.getDescription(), - ViewUtils.mapDefinition(input.getDefinition()), + ViewUtils.mapDefinition( + input.getDefinition(), context.getOperationContext().getAspectRetriever()), System.currentTimeMillis()); log.info(String.format("Successfully updated View %s with urn", urn)); return getView(context, urn, context.getAuthentication()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java index 29ceba2f1b86c..70a5ced4bfbf1 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java @@ -13,6 +13,7 @@ import com.linkedin.datahub.graphql.generated.LogicalOperator; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; import com.linkedin.metadata.query.filter.CriterionArray; @@ -25,6 +26,7 @@ import java.util.Objects; import java.util.stream.Collectors; import javax.annotation.Nonnull; +import javax.annotation.Nullable; public class ViewUtils { @@ -92,12 +94,12 @@ public static boolean canUpdateView( */ @Nonnull public static DataHubViewDefinition mapDefinition( - @Nonnull final DataHubViewDefinitionInput input) { + @Nonnull final DataHubViewDefinitionInput input, @Nullable AspectRetriever aspectRetriever) { Objects.requireNonNull(input, "input must not be null"); final DataHubViewDefinition result = new DataHubViewDefinition(); if (input.getFilter() != null) { - result.setFilter(mapFilter(input.getFilter()), SetMode.IGNORE_NULL); + result.setFilter(mapFilter(input.getFilter(), aspectRetriever), SetMode.IGNORE_NULL); } result.setEntityTypes( new StringArray( @@ -118,17 +120,19 @@ public static DataHubViewDefinition mapDefinition( * which cannot be rendered in full by the UI. We account for this on the read path by logging a * warning and returning an empty View in such cases. */ - private static Filter mapFilter(@Nonnull DataHubViewFilterInput input) { + private static Filter mapFilter( + @Nonnull DataHubViewFilterInput input, @Nullable AspectRetriever aspectRetriever) { if (LogicalOperator.AND.equals(input.getOperator())) { // AND - return buildAndFilter(input.getFilters()); + return buildAndFilter(input.getFilters(), aspectRetriever); } else { // OR - return buildOrFilter(input.getFilters()); + return buildOrFilter(input.getFilters(), aspectRetriever); } } - private static Filter buildAndFilter(@Nonnull List input) { + private static Filter buildAndFilter( + @Nonnull List input, @Nullable AspectRetriever aspectRetriever) { final Filter result = new Filter(); result.setOr( new ConjunctiveCriterionArray( @@ -137,12 +141,13 @@ private static Filter buildAndFilter(@Nonnull List input) { .setAnd( new CriterionArray( input.stream() - .map(ResolverUtils::criterionFromFilter) + .map(f -> ResolverUtils.criterionFromFilter(f, aspectRetriever)) .collect(Collectors.toList())))))); return result; } - private static Filter buildOrFilter(@Nonnull List input) { + private static Filter buildOrFilter( + @Nonnull List input, @Nullable AspectRetriever aspectRetriever) { final Filter result = new Filter(); result.setOr( new ConjunctiveCriterionArray( @@ -152,7 +157,9 @@ private static Filter buildOrFilter(@Nonnull List input) { new ConjunctiveCriterion() .setAnd( new CriterionArray( - ImmutableList.of(ResolverUtils.criterionFromFilter(filter))))) + ImmutableList.of( + ResolverUtils.criterionFromFilter( + filter, aspectRetriever))))) .collect(Collectors.toList()))); return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionMapper.java index ca13792b1e92b..a5f6cadb41566 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionMapper.java @@ -2,6 +2,8 @@ import static com.linkedin.metadata.Constants.GLOBAL_TAGS_ASPECT_NAME; +import com.linkedin.assertion.AssertionAction; +import com.linkedin.assertion.AssertionActions; import com.linkedin.assertion.AssertionInfo; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.GlobalTags; @@ -10,24 +12,41 @@ import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.Assertion; +import com.linkedin.datahub.graphql.generated.AssertionActionType; +import com.linkedin.datahub.graphql.generated.AssertionSource; +import com.linkedin.datahub.graphql.generated.AssertionSourceType; import com.linkedin.datahub.graphql.generated.AssertionStdAggregation; import com.linkedin.datahub.graphql.generated.AssertionStdOperator; import com.linkedin.datahub.graphql.generated.AssertionStdParameter; import com.linkedin.datahub.graphql.generated.AssertionStdParameterType; import com.linkedin.datahub.graphql.generated.AssertionStdParameters; import com.linkedin.datahub.graphql.generated.AssertionType; +import com.linkedin.datahub.graphql.generated.AuditStamp; +import com.linkedin.datahub.graphql.generated.CustomAssertionInfo; import com.linkedin.datahub.graphql.generated.DataPlatform; import com.linkedin.datahub.graphql.generated.DatasetAssertionInfo; import com.linkedin.datahub.graphql.generated.DatasetAssertionScope; +import com.linkedin.datahub.graphql.generated.DateInterval; import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FieldAssertionInfo; +import com.linkedin.datahub.graphql.generated.FixedIntervalSchedule; +import com.linkedin.datahub.graphql.generated.FreshnessAssertionInfo; +import com.linkedin.datahub.graphql.generated.SchemaAssertionCompatibility; +import com.linkedin.datahub.graphql.generated.SchemaAssertionField; +import com.linkedin.datahub.graphql.generated.SchemaAssertionInfo; import com.linkedin.datahub.graphql.generated.SchemaFieldRef; +import com.linkedin.datahub.graphql.generated.SqlAssertionInfo; +import com.linkedin.datahub.graphql.generated.VolumeAssertionInfo; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper; +import com.linkedin.datahub.graphql.types.dataset.mappers.SchemaFieldMapper; +import com.linkedin.datahub.graphql.types.dataset.mappers.SchemaMetadataMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.metadata.Constants; +import com.linkedin.schema.SchemaField; import java.util.Collections; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -48,6 +67,14 @@ public static Assertion map(@Nullable QueryContext context, final EntityResponse result.setInfo( mapAssertionInfo(context, new AssertionInfo(envelopedAssertionInfo.getValue().data()))); } + + final EnvelopedAspect envelopedAssertionActions = + aspects.get(Constants.ASSERTION_ACTIONS_ASPECT_NAME); + if (envelopedAssertionActions != null) { + result.setActions( + mapAssertionActions(new AssertionActions(envelopedAssertionActions.getValue().data()))); + } + final EnvelopedAspect envelopedPlatformInstance = aspects.get(Constants.DATA_PLATFORM_INSTANCE_ASPECT_NAME); if (envelopedPlatformInstance != null) { @@ -83,20 +110,103 @@ private static com.linkedin.datahub.graphql.generated.Status mapStatus(Status st return result; } - private static com.linkedin.datahub.graphql.generated.AssertionInfo mapAssertionInfo( + public static com.linkedin.datahub.graphql.generated.AssertionInfo mapAssertionInfo( @Nullable QueryContext context, final AssertionInfo gmsAssertionInfo) { final com.linkedin.datahub.graphql.generated.AssertionInfo assertionInfo = new com.linkedin.datahub.graphql.generated.AssertionInfo(); assertionInfo.setType(AssertionType.valueOf(gmsAssertionInfo.getType().name())); + + if (gmsAssertionInfo.hasLastUpdated()) { + assertionInfo.setLastUpdated( + new AuditStamp( + gmsAssertionInfo.getLastUpdated().getTime(), + gmsAssertionInfo.getLastUpdated().getActor().toString())); + } if (gmsAssertionInfo.hasDatasetAssertion()) { DatasetAssertionInfo datasetAssertion = mapDatasetAssertionInfo(context, gmsAssertionInfo.getDatasetAssertion()); assertionInfo.setDatasetAssertion(datasetAssertion); } - assertionInfo.setDescription(gmsAssertionInfo.getDescription()); + // Description + if (gmsAssertionInfo.hasDescription()) { + assertionInfo.setDescription(gmsAssertionInfo.getDescription()); + } + // FRESHNESS Assertions + if (gmsAssertionInfo.hasFreshnessAssertion()) { + FreshnessAssertionInfo freshnessAssertionInfo = + FreshnessAssertionMapper.mapFreshnessAssertionInfo( + context, gmsAssertionInfo.getFreshnessAssertion()); + assertionInfo.setFreshnessAssertion(freshnessAssertionInfo); + } + // VOLUME Assertions + if (gmsAssertionInfo.hasVolumeAssertion()) { + VolumeAssertionInfo volumeAssertionInfo = + VolumeAssertionMapper.mapVolumeAssertionInfo( + context, gmsAssertionInfo.getVolumeAssertion()); + assertionInfo.setVolumeAssertion(volumeAssertionInfo); + } + // SQL Assertions + if (gmsAssertionInfo.hasSqlAssertion()) { + SqlAssertionInfo sqlAssertionInfo = + SqlAssertionMapper.mapSqlAssertionInfo(gmsAssertionInfo.getSqlAssertion()); + assertionInfo.setSqlAssertion(sqlAssertionInfo); + } + // FIELD Assertions + if (gmsAssertionInfo.hasFieldAssertion()) { + FieldAssertionInfo fieldAssertionInfo = + FieldAssertionMapper.mapFieldAssertionInfo(context, gmsAssertionInfo.getFieldAssertion()); + assertionInfo.setFieldAssertion(fieldAssertionInfo); + } + // SCHEMA Assertions + if (gmsAssertionInfo.hasSchemaAssertion()) { + SchemaAssertionInfo schemaAssertionInfo = + mapSchemaAssertionInfo(context, gmsAssertionInfo.getSchemaAssertion()); + assertionInfo.setSchemaAssertion(schemaAssertionInfo); + } + if (gmsAssertionInfo.hasCustomAssertion()) { + CustomAssertionInfo customAssertionInfo = + mapCustomAssertionInfo(context, gmsAssertionInfo.getCustomAssertion()); + assertionInfo.setCustomAssertion(customAssertionInfo); + } + + // Source Type + if (gmsAssertionInfo.hasSource()) { + assertionInfo.setSource(mapSource(gmsAssertionInfo.getSource())); + } + + if (gmsAssertionInfo.hasExternalUrl()) { + assertionInfo.setExternalUrl(gmsAssertionInfo.getExternalUrl().toString()); + } return assertionInfo; } + private static com.linkedin.datahub.graphql.generated.AssertionActions mapAssertionActions( + final AssertionActions gmsAssertionActions) { + final com.linkedin.datahub.graphql.generated.AssertionActions result = + new com.linkedin.datahub.graphql.generated.AssertionActions(); + if (gmsAssertionActions.hasOnFailure()) { + result.setOnFailure( + gmsAssertionActions.getOnFailure().stream() + .map(AssertionMapper::mapAssertionAction) + .collect(Collectors.toList())); + } + if (gmsAssertionActions.hasOnSuccess()) { + result.setOnSuccess( + gmsAssertionActions.getOnSuccess().stream() + .map(AssertionMapper::mapAssertionAction) + .collect(Collectors.toList())); + } + return result; + } + + private static com.linkedin.datahub.graphql.generated.AssertionAction mapAssertionAction( + final AssertionAction gmsAssertionAction) { + final com.linkedin.datahub.graphql.generated.AssertionAction result = + new com.linkedin.datahub.graphql.generated.AssertionAction(); + result.setType(AssertionActionType.valueOf(gmsAssertionAction.getType().toString())); + return result; + } + private static DatasetAssertionInfo mapDatasetAssertionInfo( @Nullable QueryContext context, final com.linkedin.assertion.DatasetAssertionInfo gmsDatasetAssertion) { @@ -152,7 +262,7 @@ private static SchemaFieldRef mapDatasetSchemaField(final Urn schemaFieldUrn) { return new SchemaFieldRef(schemaFieldUrn.toString(), schemaFieldUrn.getEntityKey().get(1)); } - private static AssertionStdParameters mapParameters( + protected static AssertionStdParameters mapParameters( final com.linkedin.assertion.AssertionStdParameters params) { final AssertionStdParameters result = new AssertionStdParameters(); if (params.hasValue()) { @@ -175,5 +285,77 @@ private static AssertionStdParameter mapParameter( return result; } - private AssertionMapper() {} + protected static FixedIntervalSchedule mapFixedIntervalSchedule( + com.linkedin.assertion.FixedIntervalSchedule gmsFixedIntervalSchedule) { + FixedIntervalSchedule fixedIntervalSchedule = new FixedIntervalSchedule(); + fixedIntervalSchedule.setUnit(DateInterval.valueOf(gmsFixedIntervalSchedule.getUnit().name())); + fixedIntervalSchedule.setMultiple(gmsFixedIntervalSchedule.getMultiple()); + return fixedIntervalSchedule; + } + + private static AssertionSource mapSource(final com.linkedin.assertion.AssertionSource gmsSource) { + AssertionSource result = new AssertionSource(); + result.setType(AssertionSourceType.valueOf(gmsSource.getType().toString())); + if (gmsSource.hasCreated()) { + result.setCreated( + new AuditStamp( + gmsSource.getCreated().getTime(), gmsSource.getCreated().getActor().toString())); + } + return result; + } + + protected static com.linkedin.datahub.graphql.generated.SchemaFieldSpec mapSchemaFieldSpec( + final com.linkedin.schema.SchemaFieldSpec gmsField) { + final com.linkedin.datahub.graphql.generated.SchemaFieldSpec result = + new com.linkedin.datahub.graphql.generated.SchemaFieldSpec(); + result.setPath(gmsField.getPath()); + result.setType(gmsField.getType()); + result.setNativeType(gmsField.getNativeType()); + return result; + } + + private static SchemaAssertionInfo mapSchemaAssertionInfo( + @Nullable final QueryContext context, + final com.linkedin.assertion.SchemaAssertionInfo gmsSchemaAssertionInfo) { + SchemaAssertionInfo result = new SchemaAssertionInfo(); + result.setCompatibility( + SchemaAssertionCompatibility.valueOf(gmsSchemaAssertionInfo.getCompatibility().name())); + result.setEntityUrn(gmsSchemaAssertionInfo.getEntity().toString()); + result.setSchema( + SchemaMetadataMapper.INSTANCE.apply( + context, gmsSchemaAssertionInfo.getSchema(), gmsSchemaAssertionInfo.getEntity(), 0L)); + result.setFields( + gmsSchemaAssertionInfo.getSchema().getFields().stream() + .map(AssertionMapper::mapSchemaField) + .collect(Collectors.toList())); + return result; + } + + private static CustomAssertionInfo mapCustomAssertionInfo( + @Nullable final QueryContext context, + final com.linkedin.assertion.CustomAssertionInfo gmsCustomAssertionInfo) { + CustomAssertionInfo result = new CustomAssertionInfo(); + result.setType(gmsCustomAssertionInfo.getType()); + result.setEntityUrn(gmsCustomAssertionInfo.getEntity().toString()); + if (gmsCustomAssertionInfo.hasField()) { + result.setField(AssertionMapper.mapDatasetSchemaField(gmsCustomAssertionInfo.getField())); + } + if (gmsCustomAssertionInfo.hasLogic()) { + result.setLogic(gmsCustomAssertionInfo.getLogic()); + } + + return result; + } + + private static SchemaAssertionField mapSchemaField(final SchemaField gmsField) { + SchemaAssertionField result = new SchemaAssertionField(); + result.setPath(gmsField.getFieldPath()); + result.setType(new SchemaFieldMapper().mapSchemaFieldDataType(gmsField.getType())); + if (gmsField.hasNativeDataType()) { + result.setNativeType(gmsField.getNativeDataType()); + } + return result; + } + + protected AssertionMapper() {} } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionType.java index 0cf74439132fe..9c90478f03dc5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionType.java @@ -28,8 +28,8 @@ public class AssertionType Constants.ASSERTION_KEY_ASPECT_NAME, Constants.ASSERTION_INFO_ASPECT_NAME, Constants.DATA_PLATFORM_INSTANCE_ASPECT_NAME, - Constants.GLOBAL_TAGS_ASPECT_NAME); - + Constants.GLOBAL_TAGS_ASPECT_NAME, + Constants.ASSERTION_ACTIONS_ASPECT_NAME); private final EntityClient _entityClient; public AssertionType(final EntityClient entityClient) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/FieldAssertionMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/FieldAssertionMapper.java new file mode 100644 index 0000000000000..82d041a464c3f --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/FieldAssertionMapper.java @@ -0,0 +1,92 @@ +package com.linkedin.datahub.graphql.types.assertion; + +import com.linkedin.assertion.FieldAssertionInfo; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.AssertionStdOperator; +import com.linkedin.datahub.graphql.generated.FieldAssertionType; +import com.linkedin.datahub.graphql.generated.FieldMetricType; +import com.linkedin.datahub.graphql.generated.FieldTransformType; +import com.linkedin.datahub.graphql.generated.FieldValuesFailThresholdType; +import com.linkedin.datahub.graphql.types.dataset.mappers.DatasetFilterMapper; +import javax.annotation.Nullable; + +public class FieldAssertionMapper extends AssertionMapper { + + public static com.linkedin.datahub.graphql.generated.FieldAssertionInfo mapFieldAssertionInfo( + @Nullable final QueryContext context, final FieldAssertionInfo gmsFieldAssertionInfo) { + final com.linkedin.datahub.graphql.generated.FieldAssertionInfo result = + new com.linkedin.datahub.graphql.generated.FieldAssertionInfo(); + result.setEntityUrn(gmsFieldAssertionInfo.getEntity().toString()); + result.setType(FieldAssertionType.valueOf(gmsFieldAssertionInfo.getType().name())); + if (gmsFieldAssertionInfo.hasFilter()) { + result.setFilter(DatasetFilterMapper.map(context, gmsFieldAssertionInfo.getFilter())); + } + if (gmsFieldAssertionInfo.hasFieldValuesAssertion()) { + result.setFieldValuesAssertion( + mapFieldValuesAssertion(gmsFieldAssertionInfo.getFieldValuesAssertion())); + } + if (gmsFieldAssertionInfo.hasFieldMetricAssertion()) { + result.setFieldMetricAssertion( + mapFieldMetricAssertion(gmsFieldAssertionInfo.getFieldMetricAssertion())); + } + return result; + } + + private static com.linkedin.datahub.graphql.generated.FieldValuesAssertion + mapFieldValuesAssertion( + final com.linkedin.assertion.FieldValuesAssertion gmsFieldValuesAssertion) { + final com.linkedin.datahub.graphql.generated.FieldValuesAssertion result = + new com.linkedin.datahub.graphql.generated.FieldValuesAssertion(); + result.setField(mapSchemaFieldSpec(gmsFieldValuesAssertion.getField())); + result.setOperator(AssertionStdOperator.valueOf(gmsFieldValuesAssertion.getOperator().name())); + result.setFailThreshold( + mapFieldValuesFailThreshold(gmsFieldValuesAssertion.getFailThreshold())); + result.setExcludeNulls(gmsFieldValuesAssertion.isExcludeNulls()); + + if (gmsFieldValuesAssertion.hasTransform()) { + result.setTransform(mapFieldTransform(gmsFieldValuesAssertion.getTransform())); + } + + if (gmsFieldValuesAssertion.hasParameters()) { + result.setParameters(mapParameters(gmsFieldValuesAssertion.getParameters())); + } + return result; + } + + private static com.linkedin.datahub.graphql.generated.FieldMetricAssertion + mapFieldMetricAssertion( + final com.linkedin.assertion.FieldMetricAssertion gmsFieldMetricAssertion) { + final com.linkedin.datahub.graphql.generated.FieldMetricAssertion result = + new com.linkedin.datahub.graphql.generated.FieldMetricAssertion(); + result.setField(mapSchemaFieldSpec(gmsFieldMetricAssertion.getField())); + result.setMetric(FieldMetricType.valueOf(gmsFieldMetricAssertion.getMetric().name())); + result.setOperator(AssertionStdOperator.valueOf(gmsFieldMetricAssertion.getOperator().name())); + + if (gmsFieldMetricAssertion.hasParameters()) { + result.setParameters(mapParameters(gmsFieldMetricAssertion.getParameters())); + } + + return result; + } + + private static com.linkedin.datahub.graphql.generated.FieldTransform mapFieldTransform( + final com.linkedin.assertion.FieldTransform gmsFieldTransform) { + final com.linkedin.datahub.graphql.generated.FieldTransform result = + new com.linkedin.datahub.graphql.generated.FieldTransform(); + result.setType(FieldTransformType.valueOf(gmsFieldTransform.getType().name())); + return result; + } + + private static com.linkedin.datahub.graphql.generated.FieldValuesFailThreshold + mapFieldValuesFailThreshold( + final com.linkedin.assertion.FieldValuesFailThreshold gmsFieldValuesFailThreshold) { + final com.linkedin.datahub.graphql.generated.FieldValuesFailThreshold result = + new com.linkedin.datahub.graphql.generated.FieldValuesFailThreshold(); + result.setType( + FieldValuesFailThresholdType.valueOf(gmsFieldValuesFailThreshold.getType().name())); + result.setValue(gmsFieldValuesFailThreshold.getValue()); + return result; + } + + private FieldAssertionMapper() {} +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/FreshnessAssertionMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/FreshnessAssertionMapper.java new file mode 100644 index 0000000000000..22e1c1d8bae9e --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/FreshnessAssertionMapper.java @@ -0,0 +1,59 @@ +package com.linkedin.datahub.graphql.types.assertion; + +import com.linkedin.data.template.GetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.FreshnessAssertionInfo; +import com.linkedin.datahub.graphql.generated.FreshnessAssertionSchedule; +import com.linkedin.datahub.graphql.generated.FreshnessAssertionScheduleType; +import com.linkedin.datahub.graphql.generated.FreshnessAssertionType; +import com.linkedin.datahub.graphql.generated.FreshnessCronSchedule; +import com.linkedin.datahub.graphql.types.dataset.mappers.DatasetFilterMapper; +import javax.annotation.Nullable; + +public class FreshnessAssertionMapper extends AssertionMapper { + + public static FreshnessAssertionInfo mapFreshnessAssertionInfo( + @Nullable final QueryContext context, + final com.linkedin.assertion.FreshnessAssertionInfo gmsFreshnessAssertionInfo) { + FreshnessAssertionInfo freshnessAssertionInfo = new FreshnessAssertionInfo(); + freshnessAssertionInfo.setEntityUrn(gmsFreshnessAssertionInfo.getEntity().toString()); + freshnessAssertionInfo.setType( + FreshnessAssertionType.valueOf(gmsFreshnessAssertionInfo.getType().name())); + if (gmsFreshnessAssertionInfo.hasSchedule()) { + freshnessAssertionInfo.setSchedule( + mapFreshnessAssertionSchedule(gmsFreshnessAssertionInfo.getSchedule())); + } + if (gmsFreshnessAssertionInfo.hasFilter()) { + freshnessAssertionInfo.setFilter( + DatasetFilterMapper.map(context, gmsFreshnessAssertionInfo.getFilter())); + } + return freshnessAssertionInfo; + } + + private static FreshnessCronSchedule mapFreshnessCronSchedule( + final com.linkedin.assertion.FreshnessCronSchedule gmsCronSchedule) { + FreshnessCronSchedule cronSchedule = new FreshnessCronSchedule(); + cronSchedule.setCron(gmsCronSchedule.getCron()); + cronSchedule.setTimezone(gmsCronSchedule.getTimezone()); + cronSchedule.setWindowStartOffsetMs(gmsCronSchedule.getWindowStartOffsetMs(GetMode.NULL)); + return cronSchedule; + } + + private static FreshnessAssertionSchedule mapFreshnessAssertionSchedule( + final com.linkedin.assertion.FreshnessAssertionSchedule gmsFreshnessAssertionSchedule) { + FreshnessAssertionSchedule freshnessAssertionSchedule = new FreshnessAssertionSchedule(); + freshnessAssertionSchedule.setType( + FreshnessAssertionScheduleType.valueOf(gmsFreshnessAssertionSchedule.getType().name())); + if (gmsFreshnessAssertionSchedule.hasCron()) { + freshnessAssertionSchedule.setCron( + mapFreshnessCronSchedule(gmsFreshnessAssertionSchedule.getCron())); + } + if (gmsFreshnessAssertionSchedule.hasFixedInterval()) { + freshnessAssertionSchedule.setFixedInterval( + mapFixedIntervalSchedule(gmsFreshnessAssertionSchedule.getFixedInterval())); + } + return freshnessAssertionSchedule; + } + + private FreshnessAssertionMapper() {} +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/SqlAssertionMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/SqlAssertionMapper.java new file mode 100644 index 0000000000000..e75d2221164d4 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/SqlAssertionMapper.java @@ -0,0 +1,27 @@ +package com.linkedin.datahub.graphql.types.assertion; + +import com.linkedin.assertion.SqlAssertionInfo; +import com.linkedin.datahub.graphql.generated.AssertionStdOperator; +import com.linkedin.datahub.graphql.generated.AssertionValueChangeType; +import com.linkedin.datahub.graphql.generated.SqlAssertionType; + +public class SqlAssertionMapper extends AssertionMapper { + + public static com.linkedin.datahub.graphql.generated.SqlAssertionInfo mapSqlAssertionInfo( + final SqlAssertionInfo gmsSqlAssertionInfo) { + final com.linkedin.datahub.graphql.generated.SqlAssertionInfo result = + new com.linkedin.datahub.graphql.generated.SqlAssertionInfo(); + result.setEntityUrn(gmsSqlAssertionInfo.getEntity().toString()); + result.setType(SqlAssertionType.valueOf(gmsSqlAssertionInfo.getType().name())); + result.setStatement(gmsSqlAssertionInfo.getStatement()); + result.setOperator(AssertionStdOperator.valueOf(gmsSqlAssertionInfo.getOperator().name())); + result.setParameters(mapParameters(gmsSqlAssertionInfo.getParameters())); + if (gmsSqlAssertionInfo.hasChangeType()) { + result.setChangeType( + AssertionValueChangeType.valueOf(gmsSqlAssertionInfo.getChangeType().name())); + } + return result; + } + + private SqlAssertionMapper() {} +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/VolumeAssertionMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/VolumeAssertionMapper.java new file mode 100644 index 0000000000000..3d0294c45e520 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/VolumeAssertionMapper.java @@ -0,0 +1,115 @@ +package com.linkedin.datahub.graphql.types.assertion; + +import com.linkedin.assertion.VolumeAssertionInfo; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.AssertionStdOperator; +import com.linkedin.datahub.graphql.generated.AssertionValueChangeType; +import com.linkedin.datahub.graphql.generated.IncrementingSegmentFieldTransformerType; +import com.linkedin.datahub.graphql.generated.VolumeAssertionType; +import com.linkedin.datahub.graphql.types.dataset.mappers.DatasetFilterMapper; +import javax.annotation.Nullable; + +public class VolumeAssertionMapper extends AssertionMapper { + + public static com.linkedin.datahub.graphql.generated.VolumeAssertionInfo mapVolumeAssertionInfo( + @Nullable final QueryContext context, final VolumeAssertionInfo gmsVolumeAssertionInfo) { + final com.linkedin.datahub.graphql.generated.VolumeAssertionInfo result = + new com.linkedin.datahub.graphql.generated.VolumeAssertionInfo(); + result.setEntityUrn(gmsVolumeAssertionInfo.getEntity().toString()); + result.setType(VolumeAssertionType.valueOf(gmsVolumeAssertionInfo.getType().name())); + if (gmsVolumeAssertionInfo.hasFilter()) { + result.setFilter(DatasetFilterMapper.map(context, gmsVolumeAssertionInfo.getFilter())); + } + if (gmsVolumeAssertionInfo.hasRowCountTotal()) { + result.setRowCountTotal(mapRowCountTotal(gmsVolumeAssertionInfo.getRowCountTotal())); + } + if (gmsVolumeAssertionInfo.hasRowCountChange()) { + result.setRowCountChange(mapRowCountChange(gmsVolumeAssertionInfo.getRowCountChange())); + } + if (gmsVolumeAssertionInfo.hasIncrementingSegmentRowCountTotal()) { + result.setIncrementingSegmentRowCountTotal( + mapIncrementingSegmentRowCountTotal( + gmsVolumeAssertionInfo.getIncrementingSegmentRowCountTotal())); + } + if (gmsVolumeAssertionInfo.hasIncrementingSegmentRowCountChange()) { + result.setIncrementingSegmentRowCountChange( + mapIncrementingSegmentRowCountChange( + gmsVolumeAssertionInfo.getIncrementingSegmentRowCountChange())); + } + return result; + } + + private static com.linkedin.datahub.graphql.generated.RowCountTotal mapRowCountTotal( + final com.linkedin.assertion.RowCountTotal gmsRowCountTotal) { + final com.linkedin.datahub.graphql.generated.RowCountTotal result = + new com.linkedin.datahub.graphql.generated.RowCountTotal(); + result.setOperator(AssertionStdOperator.valueOf(gmsRowCountTotal.getOperator().name())); + result.setParameters(mapParameters(gmsRowCountTotal.getParameters())); + return result; + } + + private static com.linkedin.datahub.graphql.generated.RowCountChange mapRowCountChange( + final com.linkedin.assertion.RowCountChange gmsRowCountChange) { + final com.linkedin.datahub.graphql.generated.RowCountChange result = + new com.linkedin.datahub.graphql.generated.RowCountChange(); + result.setOperator(AssertionStdOperator.valueOf(gmsRowCountChange.getOperator().name())); + result.setParameters(mapParameters(gmsRowCountChange.getParameters())); + result.setType(AssertionValueChangeType.valueOf(gmsRowCountChange.getType().name())); + return result; + } + + private static com.linkedin.datahub.graphql.generated.IncrementingSegmentRowCountTotal + mapIncrementingSegmentRowCountTotal( + final com.linkedin.assertion.IncrementingSegmentRowCountTotal + gmsIncrementingSegmentRowCountTotal) { + final com.linkedin.datahub.graphql.generated.IncrementingSegmentRowCountTotal result = + new com.linkedin.datahub.graphql.generated.IncrementingSegmentRowCountTotal(); + result.setOperator( + AssertionStdOperator.valueOf(gmsIncrementingSegmentRowCountTotal.getOperator().name())); + result.setParameters(mapParameters(gmsIncrementingSegmentRowCountTotal.getParameters())); + result.setSegment(mapIncrementingSegmentSpec(gmsIncrementingSegmentRowCountTotal.getSegment())); + return result; + } + + private static com.linkedin.datahub.graphql.generated.IncrementingSegmentRowCountChange + mapIncrementingSegmentRowCountChange( + final com.linkedin.assertion.IncrementingSegmentRowCountChange + gmsIncrementingSegmentRowCountChange) { + final com.linkedin.datahub.graphql.generated.IncrementingSegmentRowCountChange result = + new com.linkedin.datahub.graphql.generated.IncrementingSegmentRowCountChange(); + result.setOperator( + AssertionStdOperator.valueOf(gmsIncrementingSegmentRowCountChange.getOperator().name())); + result.setParameters(mapParameters(gmsIncrementingSegmentRowCountChange.getParameters())); + result.setSegment( + mapIncrementingSegmentSpec(gmsIncrementingSegmentRowCountChange.getSegment())); + result.setType( + AssertionValueChangeType.valueOf(gmsIncrementingSegmentRowCountChange.getType().name())); + return result; + } + + private static com.linkedin.datahub.graphql.generated.IncrementingSegmentSpec + mapIncrementingSegmentSpec(final com.linkedin.assertion.IncrementingSegmentSpec gmsSegment) { + final com.linkedin.datahub.graphql.generated.IncrementingSegmentSpec result = + new com.linkedin.datahub.graphql.generated.IncrementingSegmentSpec(); + result.setField(mapSchemaFieldSpec(gmsSegment.getField())); + if (gmsSegment.hasTransformer()) { + result.setTransformer(mapIncrementingSegmentFieldTransformer(gmsSegment.getTransformer())); + } + return result; + } + + private static com.linkedin.datahub.graphql.generated.IncrementingSegmentFieldTransformer + mapIncrementingSegmentFieldTransformer( + final com.linkedin.assertion.IncrementingSegmentFieldTransformer gmsTransformer) { + final com.linkedin.datahub.graphql.generated.IncrementingSegmentFieldTransformer result = + new com.linkedin.datahub.graphql.generated.IncrementingSegmentFieldTransformer(); + result.setType( + IncrementingSegmentFieldTransformerType.valueOf(gmsTransformer.getType().name())); + if (gmsTransformer.hasNativeType()) { + result.setNativeType(gmsTransformer.getNativeType()); + } + return result; + } + + private VolumeAssertionMapper() {} +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DeprecationMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DeprecationMapper.java index 8c3d72edfed25..6959a6dcbd039 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DeprecationMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DeprecationMapper.java @@ -20,6 +20,7 @@ public Deprecation apply( @Nullable QueryContext context, @Nonnull final com.linkedin.common.Deprecation input) { final Deprecation result = new Deprecation(); result.setActor(input.getActor().toString()); + result.setActorEntity(UrnToEntityMapper.map(context, input.getActor())); result.setDeprecated(input.isDeprecated()); result.setDecommissionTime(input.getDecommissionTime()); result.setNote(input.getNote()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java index b1ce42e72482a..3c2bfd7225edf 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java @@ -1,11 +1,13 @@ package com.linkedin.datahub.graphql.types.corpuser; +import static com.linkedin.datahub.graphql.Constants.DEFAULT_PERSONA_URNS; import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; import static com.linkedin.metadata.Constants.*; import com.datahub.authorization.ConjunctivePrivilegeGroup; import com.datahub.authorization.DisjunctivePrivilegeGroup; import com.google.common.collect.ImmutableList; +import com.linkedin.common.UrnArray; import com.linkedin.common.url.Url; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; @@ -14,6 +16,8 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLException; import com.linkedin.datahub.graphql.featureflags.FeatureFlags; import com.linkedin.datahub.graphql.generated.AutoCompleteResults; import com.linkedin.datahub.graphql.generated.CorpUser; @@ -246,7 +250,20 @@ private RecordTemplate mapCorpUserEditableInfo( if (input.getEmail() != null) { result.setEmail(input.getEmail()); } - + if (input.getPlatformUrns() != null) { + result.setPlatforms( + new UrnArray( + input.getPlatformUrns().stream().map(UrnUtils::getUrn).collect(Collectors.toList()))); + } + if (input.getPersonaUrn() != null) { + if (DEFAULT_PERSONA_URNS.contains(input.getPersonaUrn())) { + result.setPersona(UrnUtils.getUrn(input.getPersonaUrn())); + } else { + throw new DataHubGraphQLException( + String.format("Provided persona urn %s does not exist", input.getPersonaUrn()), + DataHubGraphQLErrorCode.NOT_FOUND); + } + } return result; } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserEditableInfoMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserEditableInfoMapper.java index 1ff2f069b8112..38f3c75d7a9fa 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserEditableInfoMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserEditableInfoMapper.java @@ -2,7 +2,10 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.CorpUserEditableProperties; +import com.linkedin.datahub.graphql.generated.DataHubPersona; +import com.linkedin.datahub.graphql.generated.DataPlatform; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -38,6 +41,22 @@ public CorpUserEditableProperties apply( if (info.hasPictureLink()) { result.setPictureLink(info.getPictureLink().toString()); } + if (info.hasPlatforms()) { + result.setPlatforms( + info.getPlatforms().stream() + .map( + urn -> { + DataPlatform platform = new DataPlatform(); + platform.setUrn(urn.toString()); + return platform; + }) + .collect(Collectors.toList())); + } + if (info.hasPersona()) { + DataHubPersona persona = new DataHubPersona(); + persona.setUrn(info.getPersona().toString()); + result.setPersona(persona); + } return result; } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datacontract/DataContractMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datacontract/DataContractMapper.java new file mode 100644 index 0000000000000..1fe65beed6e92 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datacontract/DataContractMapper.java @@ -0,0 +1,112 @@ +package com.linkedin.datahub.graphql.types.datacontract; + +import com.linkedin.datahub.graphql.generated.Assertion; +import com.linkedin.datahub.graphql.generated.DataContract; +import com.linkedin.datahub.graphql.generated.DataContractProperties; +import com.linkedin.datahub.graphql.generated.DataContractState; +import com.linkedin.datahub.graphql.generated.DataContractStatus; +import com.linkedin.datahub.graphql.generated.DataQualityContract; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FreshnessContract; +import com.linkedin.datahub.graphql.generated.SchemaContract; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.metadata.Constants; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +public class DataContractMapper { + + public static DataContract mapContract(@Nonnull final EntityResponse entityResponse) { + final DataContract result = new DataContract(); + final EnvelopedAspectMap aspects = entityResponse.getAspects(); + + result.setUrn(entityResponse.getUrn().toString()); + result.setType(EntityType.DATA_CONTRACT); + + final EnvelopedAspect dataContractProperties = + aspects.get(Constants.DATA_CONTRACT_PROPERTIES_ASPECT_NAME); + if (dataContractProperties != null) { + result.setProperties( + mapProperties( + new com.linkedin.datacontract.DataContractProperties( + dataContractProperties.getValue().data()))); + } else { + throw new RuntimeException( + String.format("Data Contract does not exist!. urn: %s", entityResponse.getUrn())); + } + + final EnvelopedAspect dataContractStatus = + aspects.get(Constants.DATA_CONTRACT_STATUS_ASPECT_NAME); + if (dataContractStatus != null) { + result.setStatus( + mapStatus( + new com.linkedin.datacontract.DataContractStatus( + dataContractStatus.getValue().data()))); + } + + return result; + } + + private static DataContractProperties mapProperties( + final com.linkedin.datacontract.DataContractProperties properties) { + final DataContractProperties result = new DataContractProperties(); + result.setEntityUrn(properties.getEntity().toString()); + if (properties.hasSchema()) { + result.setSchema( + properties.getSchema().stream() + .map(DataContractMapper::mapSchemaContract) + .collect(Collectors.toList())); + } + if (properties.hasFreshness()) { + result.setFreshness( + properties.getFreshness().stream() + .map(DataContractMapper::mapFreshnessContract) + .collect(Collectors.toList())); + } + if (properties.hasDataQuality()) { + result.setDataQuality( + properties.getDataQuality().stream() + .map(DataContractMapper::mapDataQualityContract) + .collect(Collectors.toList())); + } + return result; + } + + private static DataContractStatus mapStatus( + final com.linkedin.datacontract.DataContractStatus status) { + final DataContractStatus result = new DataContractStatus(); + result.setState(DataContractState.valueOf(status.getState().toString())); + return result; + } + + private static SchemaContract mapSchemaContract( + final com.linkedin.datacontract.SchemaContract schemaContract) { + final SchemaContract result = new SchemaContract(); + final Assertion partialAssertion = new Assertion(); + partialAssertion.setUrn(schemaContract.getAssertion().toString()); + result.setAssertion(partialAssertion); + return result; + } + + private static FreshnessContract mapFreshnessContract( + final com.linkedin.datacontract.FreshnessContract freshnessContract) { + final FreshnessContract result = new FreshnessContract(); + final Assertion partialAssertion = new Assertion(); + partialAssertion.setUrn(freshnessContract.getAssertion().toString()); + result.setAssertion(partialAssertion); + return result; + } + + private static DataQualityContract mapDataQualityContract( + final com.linkedin.datacontract.DataQualityContract qualityContract) { + final DataQualityContract result = new DataQualityContract(); + final Assertion partialAssertion = new Assertion(); + partialAssertion.setUrn(qualityContract.getAssertion().toString()); + result.setAssertion(partialAssertion); + return result; + } + + private DataContractMapper() {} +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datacontract/DataContractType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datacontract/DataContractType.java new file mode 100644 index 0000000000000..7f1756610baf7 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datacontract/DataContractType.java @@ -0,0 +1,84 @@ +package com.linkedin.datahub.graphql.types.datacontract; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DataContract; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import graphql.execution.DataFetcherResult; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +public class DataContractType + implements com.linkedin.datahub.graphql.types.EntityType { + + static final Set ASPECTS_TO_FETCH = + ImmutableSet.of( + Constants.DATA_CONTRACT_KEY_ASPECT_NAME, + Constants.DATA_CONTRACT_PROPERTIES_ASPECT_NAME, + Constants.DATA_CONTRACT_STATUS_ASPECT_NAME); + private final EntityClient _entityClient; + + public DataContractType(final EntityClient entityClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + } + + @Override + public EntityType type() { + return EntityType.DATA_CONTRACT; + } + + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + + @Override + public Class objectClass() { + return DataContract.class; + } + + @Override + public List> batchLoad( + @Nonnull List urns, @Nonnull QueryContext context) throws Exception { + final List dataContractUrns = + urns.stream().map(UrnUtils::getUrn).collect(Collectors.toList()); + + try { + final Map entities = + _entityClient.batchGetV2( + context.getOperationContext(), + Constants.DATA_CONTRACT_ENTITY_NAME, + new HashSet<>(dataContractUrns), + ASPECTS_TO_FETCH); + + final List gmsResults = new ArrayList<>(); + for (Urn urn : dataContractUrns) { + gmsResults.add(entities.getOrDefault(urn, null)); + } + return gmsResults.stream() + .map( + gmsResult -> + gmsResult == null + ? null + : DataFetcherResult.newResult() + .data(DataContractMapper.mapContract(gmsResult)) + .build()) + .collect(Collectors.toList()); + } catch (Exception e) { + throw new RuntimeException("Failed to batch load Data Contracts", e); + } + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/AssertionRunEventMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/AssertionRunEventMapper.java index e63335beef9c1..89e636e606601 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/AssertionRunEventMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/AssertionRunEventMapper.java @@ -38,6 +38,7 @@ public com.linkedin.datahub.graphql.generated.AssertionRunEvent apply( final com.linkedin.datahub.graphql.generated.AssertionRunEvent assertionRunEvent = new com.linkedin.datahub.graphql.generated.AssertionRunEvent(); + assertionRunEvent.setLastObservedMillis(envelopedAspect.getSystemMetadata().getLastObserved()); assertionRunEvent.setTimestampMillis(gmsAssertionRunEvent.getTimestampMillis()); assertionRunEvent.setAssertionUrn(gmsAssertionRunEvent.getAssertionUrn().toString()); assertionRunEvent.setAsserteeUrn(gmsAssertionRunEvent.getAsserteeUrn().toString()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java index a2cc9d5a66edd..3674186ac23fe 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java @@ -51,7 +51,7 @@ public SchemaField apply( return result; } - private SchemaFieldDataType mapSchemaFieldDataType( + public SchemaFieldDataType mapSchemaFieldDataType( @Nonnull final com.linkedin.schema.SchemaFieldDataType dataTypeUnion) { final com.linkedin.schema.SchemaFieldDataType.Type type = dataTypeUnion.getType(); if (type.isBytesType()) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeMapper.java index ffb14df5e800b..26835f9e57dcd 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeMapper.java @@ -48,6 +48,7 @@ public class EntityTypeMapper { .put(EntityType.BUSINESS_ATTRIBUTE, Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME) .put(EntityType.QUERY, Constants.QUERY_ENTITY_NAME) .put(EntityType.POST, Constants.POST_ENTITY_NAME) + .put(EntityType.FORM, Constants.FORM_ENTITY_NAME) .build(); private static final Map ENTITY_NAME_TO_TYPE = diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelationship/CreateERModelRelationshipResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelationship/CreateERModelRelationshipResolver.java index 61896ed1a0659..cafd0b5ab082b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelationship/CreateERModelRelationshipResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ermodelrelationship/CreateERModelRelationshipResolver.java @@ -54,14 +54,15 @@ public CompletableFuture get(DataFetchingEnvironment enviro highDataset = source; } // The following sequence mimics datahub.emitter.mce_builder.datahub_guid + // Keys have to be in alphabetical order - Destination, ERModelRelationName and Source String ermodelrelationKey = - "{\"Source\":\"" + "{\"Destination\":\"" + lowDataset - + "\",\"Destination\":\"" - + highDataset + "\",\"ERModelRelationName\":\"" + ermodelrelationName + + "\",\"Source\":\"" + + highDataset + "\"}"; byte[] mybytes = ermodelrelationKey.getBytes(StandardCharsets.UTF_8); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/UrnSearchAcrossLineageResultsMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/UrnSearchAcrossLineageResultsMapper.java index ca363deb90c4d..d0c5605976d63 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/UrnSearchAcrossLineageResultsMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/UrnSearchAcrossLineageResultsMapper.java @@ -71,6 +71,7 @@ private SearchAcrossLineageResult mapResult( .setDegrees(new ArrayList<>(searchEntity.getDegrees())) .setExplored(Boolean.TRUE.equals(searchEntity.isExplored())) .setIgnoredAsHop(Boolean.TRUE.equals(searchEntity.isIgnoredAsHop())) + .setTruncatedChildren(Boolean.TRUE.equals(searchEntity.isTruncatedChildren())) .build(); } } diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index d84a86a3bedd3..b3a965981c366 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -492,6 +492,11 @@ type FeatureFlagsConfig { Whether business attribute entity should be shown """ businessAttributeEntityEnabled: Boolean! + + """ + Whether data contracts should be enabled + """ + dataContractsEnabled: Boolean! } """ diff --git a/datahub-graphql-core/src/main/resources/assertions.graphql b/datahub-graphql-core/src/main/resources/assertions.graphql new file mode 100644 index 0000000000000..ff182089ad7ff --- /dev/null +++ b/datahub-graphql-core/src/main/resources/assertions.graphql @@ -0,0 +1,1066 @@ +extend type Mutation { + """ + Upsert a Custom Assertion + """ + upsertCustomAssertion( + """ + Urn of custom assertion. If not provided, one will be generated. + """ + urn: String + + """ + Input for upserting a custom assertion. + """ + input: UpsertCustomAssertionInput! + ): Assertion! + + """ + Report result for an assertion + """ + reportAssertionResult( + """ + Urn of custom assertion. + """ + urn: String! + + """ + Input for reporting result of the assertion + """ + result: AssertionResultInput! + ): Boolean! +} + +""" +Input for upserting a Custom Assertion. +""" +input UpsertCustomAssertionInput { + """ + The entity targeted by this assertion. + """ + entityUrn: String! + + """ + The type of the custom assertion. + """ + type: String! + + """ + The description of this assertion. + """ + description: String! + + """ + The dataset field targeted by this assertion, if any. + """ + fieldPath: String + + """ + The external Platform associated with the assertion + """ + platform: PlatformInput! + + """ + Native platform URL of the Assertion + """ + externalUrl: String + + """ + Logic comprising a raw, unstructured assertion. for example - custom SQL query for the assertion. + """ + logic: String + +} + +""" +Input for reporting result of the assertion +""" +input AssertionResultInput { + """ + Optional: Provide a timestamp associated with the run event. If not provided, one will be generated for you based + on the current time. + """ + timestampMillis: Long + + """ + The final result of assertion, e.g. either SUCCESS or FAILURE. + """ + type: AssertionResultType! + + """ + Additional metadata representing about the native results of the assertion. + These will be displayed alongside the result. + It should be used to capture additional context that is useful for the user. + """ + properties: [StringMapEntryInput!] + + """ + Native platform URL of the Assertion Run Event + """ + externalUrl: String + + """ + Error details, if type is ERROR + """ + error: AssertionResultErrorInput +} + +""" +Input for reporting an Error during Assertion Run +""" +input AssertionResultErrorInput { + """ + The type of error encountered + """ + type: AssertionResultErrorType! + + """ + The error message with details of error encountered + """ + message: String! +} +""" +Input representing A Data Platform +""" +input PlatformInput { + """ + Urn of platform + """ + urn: String + + """ + Name of platform + """ + name: String +} + +""" +Defines a schema field, each with a specified path and type. +""" +type SchemaAssertionField { + """ + The standard V1 path of the field within the schema. + """ + path: String! + + """ + The std type of the field + """ + type: SchemaFieldDataType! + + """ + Optional: The specific native or standard type of the field. + """ + nativeType: String +} + +""" +Defines the required compatibility level for the schema assertion to pass. +""" +enum SchemaAssertionCompatibility { + """ + The schema must be exactly the same as the expected schema. + """ + EXACT_MATCH + + """ + The schema must be a superset of the expected schema. + """ + SUPERSET + + """ + The schema must be a subset of the expected schema. + """ + SUBSET +} + +""" +The source of an assertion +""" +enum AssertionSourceType { + """ + The assertion was defined natively on DataHub by a user. + """ + NATIVE + """ + The assertion was defined and managed externally of DataHub. + """ + EXTERNAL + """ + The assertion was inferred, e.g. from offline AI / ML models. + """ + INFERRED +} + +""" +The type of an Freshness assertion +""" +enum FreshnessAssertionType { + """ + An assertion defined against a Dataset Change Operation - insert, update, delete, etc + """ + DATASET_CHANGE + """ + An assertion defined against a Data Job run + """ + DATA_JOB_RUN +} + +extend type AssertionInfo { + """ + Information about an Freshness Assertion + """ + freshnessAssertion: FreshnessAssertionInfo + + """ + Information about an Volume Assertion + """ + volumeAssertion: VolumeAssertionInfo + + """ + Information about a SQL Assertion + """ + sqlAssertion: SqlAssertionInfo + + """ + Information about a Field Assertion + """ + fieldAssertion: FieldAssertionInfo + + """ + Schema assertion, e.g. defining the expected structure for an asset. + """ + schemaAssertion: SchemaAssertionInfo + + """ + Information about Custom assertion + """ + customAssertion: CustomAssertionInfo + + """ + The source or origin of the Assertion definition. + """ + source: AssertionSource + + """ + The time that the status last changed and the actor who changed it + """ + lastUpdated: AuditStamp +} + +extend type Assertion { + """ + The actions associated with the Assertion + """ + actions: AssertionActions +} + +""" +Some actions associated with an assertion +""" +type AssertionActions { + """ + Actions to be executed on successful assertion run. + """ + onSuccess: [AssertionAction!]! + + """ + Actions to be executed on failed assertion run. + """ + onFailure: [AssertionAction!]! +} + +""" +An action associated with an assertion +""" +type AssertionAction { + """ + The type of the actions + """ + type: AssertionActionType! +} + + +""" +The type of the Action +""" +enum AssertionActionType { + """ + Raise an incident. + """ + RAISE_INCIDENT + """ + Resolve open incidents related to the assertion. + """ + RESOLVE_INCIDENT +} + + +""" +Information about an Freshness assertion. +""" +type FreshnessAssertionInfo { + """ + The urn of the entity that the Freshness assertion is related to + """ + entityUrn: String! + + """ + The type of the Freshness Assertion + """ + type: FreshnessAssertionType! + + """ + Produce FAIL Assertion Result if the asset is not updated on the cadence and within the time range described by the schedule. + """ + schedule: FreshnessAssertionSchedule! + + """ + A filter applied when querying an external Dataset or Table + """ + filter: DatasetFilter +} + +""" +Attributes defining a single Freshness schedule. +""" +type FreshnessAssertionSchedule { + """ + The type of schedule + """ + type: FreshnessAssertionScheduleType! + + """ + A cron schedule. This is populated if the type is CRON. + """ + cron: FreshnessCronSchedule + + """ + A fixed interval schedule. This is populated if the type is FIXED_INTERVAL. + """ + fixedInterval: FixedIntervalSchedule +} + +""" +The type of an Freshness assertion +""" +enum FreshnessAssertionScheduleType { + """ + An schedule based on a CRON schedule representing the expected event times. + """ + CRON + + """ + A scheduled based on a recurring fixed schedule which is used to compute the expected operation window. E.g. "every 24 hours". + """ + FIXED_INTERVAL + + """ + A schedule computed based on when the assertion was last evaluated, to the current moment in time. + """ + SINCE_THE_LAST_CHECK +} + +""" +A cron-formatted schedule +""" +type FreshnessCronSchedule { + """ + A cron-formatted execution interval, as a cron string, e.g. 1 * * * * + """ + cron: String! + + """ + Timezone in which the cron interval applies, e.g. America/Los Angeles + """ + timezone: String! + + """ + An optional offset in milliseconds to SUBTRACT from the timestamp generated by the cron schedule + to generate the lower bounds of the "Freshness window", or the window of time in which an event must have occurred in order for the Freshness + to be considering passing. + If left empty, the start of the Freshness window will be the _end_ of the previously evaluated Freshness window. + """ + windowStartOffsetMs: Long +} + +""" +A fixed interval schedule. +""" +type FixedIntervalSchedule { + """ + Interval unit such as minute/hour/day etc. + """ + unit: DateInterval! + + """ + How many units. Defaults to 1. + """ + multiple: Int! +} + +""" +The source of an Assertion +""" +type AssertionSource { + """ + The source type + """ + type: AssertionSourceType! + """ + The time at which the assertion was initially created and the actor who created it + """ + created: AuditStamp +} + +""" +Information about the field to use in an assertion +""" +type SchemaFieldSpec { + """ + The field path + """ + path: String! + + """ + The DataHub standard schema field type. + """ + type: String! + + """ + The native field type + """ + nativeType: String! +} + +""" +An enum to represent a type of change in an assertion value, metric, or measurement. +""" +enum AssertionValueChangeType { + """ + A change that is defined in absolute terms. + """ + ABSOLUTE + + """ + A change that is defined in relative terms using percentage change + from the original value. + """ + PERCENTAGE +} + +""" +A type of volume (row count) assertion +""" +enum VolumeAssertionType { + """ + A volume assertion that is evaluated against the total row count of a dataset. + """ + ROW_COUNT_TOTAL + + """ + A volume assertion that is evaluated against an incremental row count of a dataset, + or a row count change. + """ + ROW_COUNT_CHANGE + + """ + A volume assertion that checks the latest "segment" in a table based on an incrementing + column to check whether it's row count falls into a particular range. + This can be used to monitor the row count of an incrementing date-partition column segment. + """ + INCREMENTING_SEGMENT_ROW_COUNT_TOTAL + + """ + A volume assertion that compares the row counts in neighboring "segments" or "partitions" + of an incrementing column. This can be used to track changes between subsequent date partition + in a table, for example. + """ + INCREMENTING_SEGMENT_ROW_COUNT_CHANGE +} + +""" +Attributes defining an ROW_COUNT_TOTAL volume assertion. +""" +type RowCountTotal { + """ + The operator you'd like to apply. + Note that only numeric operators are valid inputs: + GREATER_THAN, GREATER_THAN_OR_EQUAL_TO, EQUAL_TO, LESS_THAN, LESS_THAN_OR_EQUAL_TO, + BETWEEN. + """ + operator: AssertionStdOperator! + + """ + The parameters you'd like to provide as input to the operator. + Note that only numeric parameter types are valid inputs: NUMBER. + """ + parameters: AssertionStdParameters! +} + +""" +Attributes defining an ROW_COUNT_CHANGE volume assertion. +""" +type RowCountChange { + """ + The type of the value used to evaluate the assertion: a fixed absolute value or a relative percentage. + """ + type: AssertionValueChangeType! + + """ + The operator you'd like to apply. + Note that only numeric operators are valid inputs: + GREATER_THAN, GREATER_THAN_OR_EQUAL_TO, EQUAL_TO, LESS_THAN, LESS_THAN_OR_EQUAL_TO, + BETWEEN. + """ + operator: AssertionStdOperator! + + """ + The parameters you'd like to provide as input to the operator. + Note that only numeric parameter types are valid inputs: NUMBER. + """ + parameters: AssertionStdParameters! +} + +""" +Attributes defining an INCREMENTING_SEGMENT_ROW_COUNT_TOTAL volume assertion. +""" +type IncrementingSegmentRowCountTotal { + """ + A specification of how the 'segment' can be derived using a column and an optional transformer function. + """ + segment: IncrementingSegmentSpec! + + """ + The operator you'd like to apply. + Note that only numeric operators are valid inputs: + GREATER_THAN, GREATER_THAN_OR_EQUAL_TO, EQUAL_TO, LESS_THAN, LESS_THAN_OR_EQUAL_TO, + BETWEEN. + """ + operator: AssertionStdOperator! + + """ + The parameters you'd like to provide as input to the operator. + Note that only numeric parameter types are valid inputs: NUMBER. + """ + parameters: AssertionStdParameters! +} + +""" +Attributes defining an INCREMENTING_SEGMENT_ROW_COUNT_CHANGE volume assertion. +""" +type IncrementingSegmentRowCountChange { + """ + A specification of how the 'segment' can be derived using a column and an optional transformer function. + """ + segment: IncrementingSegmentSpec! + + """ + The type of the value used to evaluate the assertion: a fixed absolute value or a relative percentage. + """ + type: AssertionValueChangeType! + + """ + The operator you'd like to apply to the row count value + Note that only numeric operators are valid inputs: + GREATER_THAN, GREATER_THAN_OR_EQUAL_TO, EQUAL_TO, LESS_THAN, LESS_THAN_OR_EQUAL_TO, + BETWEEN. + """ + operator: AssertionStdOperator! + + """ + The parameters you'd like to provide as input to the operator. + Note that only numeric parameter types are valid inputs: NUMBER. + """ + parameters: AssertionStdParameters! +} + +""" +Core attributes required to identify an incrementing segment in a table. This type is mainly useful +for tables that constantly increase with new rows being added on a particular cadence (e.g. fact or event tables). + +An incrementing segment represents a logical chunk of data which is INSERTED +into a dataset on a regular interval, along with the presence of a constantly-incrementing column +value such as an event time, date partition, or last modified column. + +An incrementing segment is principally identified by 2 key attributes combined: + +1. A field or column that represents the incrementing value. New rows that are inserted will be identified using this column. + Note that the value of this column may not by itself represent the "bucket" or the "segment" in which the row falls. + +2. [Optional] An transformer function that may be applied to the selected column value in order + to obtain the final "segment identifier" or "bucket identifier". Rows that have the same value after applying the transformation + will be grouped into the same segment, using which the final value (e.g. row count) will be determined. +""" +type IncrementingSegmentSpec { + """ + The field to use to generate segments. It must be constantly incrementing as new rows are inserted. + """ + field: SchemaFieldSpec! + + """ + Optional transformer function to apply to the field in order to obtain the final segment or bucket identifier. + If not provided, then no operator will be applied to the field. (identity function) + """ + transformer: IncrementingSegmentFieldTransformer +} + +""" +The definition of the transformer function that should be applied to a given field / column value in a dataset +in order to determine the segment or bucket that it belongs to, which in turn is used to evaluate +volume assertions. +""" +type IncrementingSegmentFieldTransformer { + """ + The 'standard' operator type. Note that not all source systems will support all operators. + """ + type: IncrementingSegmentFieldTransformerType! + + """ + The 'native' transformer type, useful as a back door if a custom transformer is required. + This field is required if the type is NATIVE. + """ + nativeType: String +} + +""" +The 'standard' transformer type. Note that not all source systems will support all operators. +""" +enum IncrementingSegmentFieldTransformerType { + """ + Rounds a timestamp (in seconds) down to the start of the month. + """ + TIMESTAMP_MS_TO_MINUTE + + """ + Rounds a timestamp (in milliseconds) down to the nearest hour. + """ + TIMESTAMP_MS_TO_HOUR + + """ + Rounds a timestamp (in milliseconds) down to the start of the day. + """ + TIMESTAMP_MS_TO_DATE + + """ + Rounds a timestamp (in milliseconds) down to the start of the month + """ + TIMESTAMP_MS_TO_MONTH + + """ + Rounds a timestamp (in milliseconds) down to the start of the year + """ + TIMESTAMP_MS_TO_YEAR + + """ + Rounds a numeric value down to the nearest integer. + """ + FLOOR + + """ + Rounds a numeric value up to the nearest integer. + """ + CEILING + + """ + A backdoor to provide a native operator type specific to a given source system like + Snowflake, Redshift, BQ, etc. + """ + NATIVE +} + +""" +A definition of a Volume (row count) assertion. +""" +type VolumeAssertionInfo { + """ + The entity targeted by this Volume check. + """ + entityUrn: String! + + """ + The type of the freshness assertion being monitored. + """ + type: VolumeAssertionType! + + """ + Produce FAILURE Assertion Result if the row count of the asset does not meet specific requirements. + Required if type is 'ROW_COUNT_TOTAL'. + """ + rowCountTotal: RowCountTotal + + """ + Produce FAILURE Assertion Result if the row count delta of the asset does not meet specific requirements. + Required if type is 'ROW_COUNT_CHANGE'. + """ + rowCountChange: RowCountChange + + """ + Produce FAILURE Assertion Result if the latest incrementing segment row count total of the asset + does not meet specific requirements. Required if type is 'INCREMENTING_SEGMENT_ROW_COUNT_TOTAL'. + """ + incrementingSegmentRowCountTotal: IncrementingSegmentRowCountTotal + + """ + Produce FAILURE Assertion Result if the incrementing segment row count delta of the asset + does not meet specific requirements. Required if type is 'INCREMENTING_SEGMENT_ROW_COUNT_CHANGE'. + """ + incrementingSegmentRowCountChange: IncrementingSegmentRowCountChange + + """ + A definition of the specific filters that should be applied, when performing monitoring. + If not provided, there is no filter, and the full table is under consideration. + """ + filter: DatasetFilter +} + +""" +The type of the SQL assertion being monitored. +""" +enum SqlAssertionType { + """ + A SQL Metric Assertion, e.g. one based on a numeric value returned by an arbitrary SQL query. + """ + METRIC + + """ + A SQL assertion that is evaluated against the CHANGE in a metric assertion over time. + """ + METRIC_CHANGE +} + +""" +Attributes defining a SQL Assertion +""" +type SqlAssertionInfo { + """ + The type of the SQL assertion being monitored. + """ + type: SqlAssertionType! + + """ + The entity targeted by this SQL check. + """ + entityUrn: String! + + """ + The SQL statement to be executed when evaluating the assertion. + """ + statement: String! + + """ + The type of the value used to evaluate the assertion: a fixed absolute value or a relative percentage. + Required if the type is METRIC_CHANGE. + """ + changeType: AssertionValueChangeType + + """ + The operator you'd like to apply to the result of the SQL query. + """ + operator: AssertionStdOperator! + + """ + The parameters you'd like to provide as input to the operator. + """ + parameters: AssertionStdParameters! +} + +""" +The type of a Field assertion +""" +enum FieldAssertionType { + """ + An assertion used to validate the values contained with a field / column given a set of rows. + """ + FIELD_VALUES + + """ + An assertion used to validate the value of a common field / column metric (e.g. aggregation) + such as null count + percentage, min, max, median, and more. + """ + FIELD_METRIC +} + +""" +The type of the Field Transform +""" +enum FieldTransformType { + """ + Obtain the length of a string field / column (applicable to string types) + """ + LENGTH +} + +""" +The type of failure threshold. +""" +enum FieldValuesFailThresholdType { + """ + The maximum number of column values (i.e. rows) that are allowed + to fail the defined expectations before the assertion officially fails. + """ + COUNT + + """ + The maximum percentage of rows that are allowed + to fail the defined column expectations before the assertion officially fails. + """ + PERCENTAGE +} + +""" +A standard metric that can be derived from the set of values +for a specific field / column of a dataset / table. +""" +enum FieldMetricType { + """ + The number of unique values found in the column value set + """ + UNIQUE_COUNT + + """ + The percentage of unique values to total rows for the dataset + """ + UNIQUE_PERCENTAGE + + """ + The number of null values found in the column value set + """ + NULL_COUNT + + """ + The percentage of null values to total rows for the dataset + """ + NULL_PERCENTAGE + + """ + The minimum value in the column set (applies to numeric columns) + """ + MIN + + """ + The maximum value in the column set (applies to numeric columns) + """ + MAX + + """ + The mean length found in the column set (applies to numeric columns) + """ + MEAN + + """ + The median length found in the column set (applies to numeric columns) + """ + MEDIAN + + """ + The stddev length found in the column set (applies to numeric columns) + """ + STDDEV + + """ + The number of negative values found in the value set (applies to numeric columns) + """ + NEGATIVE_COUNT + + """ + The percentage of negative values to total rows for the dataset (applies to numeric columns) + """ + NEGATIVE_PERCENTAGE + + """ + The number of zero values found in the value set (applies to numeric columns) + """ + ZERO_COUNT + + """ + The percentage of zero values to total rows for the dataset (applies to numeric columns) + """ + ZERO_PERCENTAGE + + """ + The minimum length found in the column set (applies to string columns) + """ + MIN_LENGTH + + """ + The maximum length found in the column set (applies to string columns) + """ + MAX_LENGTH + + """ + The number of empty string values found in the value set (applies to string columns). + Note: This is a completely different metric different from NULL_COUNT! + """ + EMPTY_COUNT + + """ + The percentage of empty string values to total rows for the dataset (applies to string columns). + Note: This is a completely different metric different from NULL_PERCENTAGE! + """ + EMPTY_PERCENTAGE +} + +""" +A definition of a Field (Column) assertion. +""" +type FieldAssertionInfo { + """ + The type of the field assertion being monitored. + """ + type: FieldAssertionType! + + """ + The entity targeted by this Field check. + """ + entityUrn: String! + + """ + The definition of an assertion that validates individual values of a field / column for a set of rows. + """ + fieldValuesAssertion: FieldValuesAssertion + + """ + The definition of an assertion that validates a common metric obtained about a field / column for a set of rows. + """ + fieldMetricAssertion: FieldMetricAssertion + + """ + A definition of the specific filters that should be applied, when performing monitoring. + If not provided, there is no filter, and the full table is under consideration. + """ + filter: DatasetFilter +} + +""" +A definition of a Field Values assertion. +""" +type FieldValuesAssertion { + """ + The field under evaluation. + """ + field: SchemaFieldSpec! + + """ + An optional transform to apply to field values before evaluating the operator. + """ + transform: FieldTransform + + """ + The predicate to evaluate against a single value of the field. + Depending on the operator, parameters may be required + """ + operator: AssertionStdOperator! + + """ + Standard parameters required for the assertion. + """ + parameters: AssertionStdParameters + + """ + Additional customization about when the assertion should be officially considered failing. + """ + failThreshold: FieldValuesFailThreshold! + + """ + Whether to ignore or allow nulls when running the values assertion. + """ + excludeNulls: Boolean! +} + +""" +Definition of a transform applied to the values of a column / field. +""" +type FieldTransform { + """ + The type of the field transform. + """ + type: FieldTransformType! +} + +type FieldValuesFailThreshold { + """ + The type of failure threshold. + """ + type: FieldValuesFailThresholdType! + + """ + The value of the threshold, either representing a count or percentage. + """ + value: Long! +} + +""" +A definition of a Field Metric assertion. +""" +type FieldMetricAssertion { + """ + The field under evaluation + """ + field: SchemaFieldSpec! + + """ + The specific metric to assert against. + """ + metric: FieldMetricType! + + """ + The predicate to evaluate against the metric for the field / column. + """ + operator: AssertionStdOperator! + + """ + Standard parameters required for the assertion. + """ + parameters: AssertionStdParameters +} + +""" +Information about an Schema assertion +""" +type SchemaAssertionInfo { + """ + The entity targeted by this schema assertion. + """ + entityUrn: String! + + """ + A single field in the schema assertion. + """ + fields: [SchemaAssertionField!]! + + """ + A definition of the expected structure for the asset + Deprecated! Use the simpler 'fields' instead. + """ + schema: SchemaMetadata + + """ + The compatibility level required for the assertion to pass. + """ + compatibility: SchemaAssertionCompatibility! +} + +""" +Information about a custom assertion +""" +type CustomAssertionInfo { + """ + The type of custom assertion. + """ + type: String! + + """ + The entity targeted by this custom assertion. + """ + entityUrn: String! + + """ + The field serving as input to the assertion, if any. + """ + field: SchemaFieldRef + + """ + Logic comprising a raw, unstructured assertion. + """ + logic: String +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/resources/contract.graphql b/datahub-graphql-core/src/main/resources/contract.graphql new file mode 100644 index 0000000000000..27d6510c89c24 --- /dev/null +++ b/datahub-graphql-core/src/main/resources/contract.graphql @@ -0,0 +1,183 @@ +extend type Mutation { + """ + Create or update a data contract for a given dataset. Requires the "Edit Data Contract" privilege for the provided dataset. + """ + upsertDataContract(urn: String, input: UpsertDataContractInput!): DataContract! +} + +extend type Dataset { + """ + An optional Data Contract defined for the Dataset. + """ + contract: DataContract +} + +""" +A Data Contract Entity. A Data Contract is a verifiable group of assertions regarding various aspects of the data: its freshness (sla), +schema, and data quality or validity. This group of assertions represents a data owner's commitment to producing data that confirms to the agreed +upon contract. Each dataset can have a single contract. The contract can be in a "passing" or "violating" state, depending +on whether the assertions that compose the contract are passing or failing. +Note that the data contract entity is currently in early preview (beta). +""" +type DataContract implements Entity { + """ + A primary key of the data contract + """ + urn: String! + + """ + The standard entity type + """ + type: EntityType! + + """ + Properties describing the data contract + """ + properties: DataContractProperties + + """ + The status of the data contract + """ + status: DataContractStatus + + """ + List of relationships between the source Entity and some destination entities with a given types + """ + relationships(input: RelationshipsInput!): EntityRelationshipsResult +} + +type DataContractProperties { + """ + The urn of the related entity, e.g. the Dataset today. In the future, we may support additional contract entities. + """ + entityUrn: String! + + """ + The Freshness (SLA) portion of the contract. + As of today, it is expected that there will not be more than 1 Freshness contract. If there are, only the first will be displayed. + """ + freshness: [FreshnessContract!] + + """ + The schema / structural portion of the contract. + As of today, it is expected that there will not be more than 1 Schema contract. If there are, only the first will be displayed. + """ + schema: [SchemaContract!] + + """ + A set of data quality related contracts, e.g. table and column-level contract constraints. + """ + dataQuality: [DataQualityContract!] +} + +""" +The state of the data contract +""" +enum DataContractState { + """ + The data contract is active. + """ + ACTIVE + + """ + The data contract is pending. Note that this symbol is currently experimental. + """ + PENDING +} + +type DataContractStatus { + """ + The state of the data contract + """ + state: DataContractState! +} + +type DataQualityContract { + """ + The assertion representing the schema contract. + """ + assertion: Assertion! +} + +type SchemaContract { + """ + The assertion representing the schema contract. + """ + assertion: Assertion! +} + +type FreshnessContract { + """ + The assertion representing the Freshness contract. + """ + assertion: Assertion! +} + +""" +Input required to upsert a Data Contract entity for an asset +""" +input UpsertDataContractInput { + """ + The urn of the related entity. Dataset is the only entity type supported today. + """ + entityUrn: String! + + """ + The Freshness / Freshness portion of the contract. If not provided, this will be set to none. + For Dataset Contracts, it is expected that there will not be more than 1 Freshness contract. If there are, only the first will be displayed. + """ + freshness: [FreshnessContractInput!] + + """ + The schema / structural portion of the contract. If not provided, this will be set to none. + For Dataset Contracts, it is expected that there will not be more than 1 Schema contract. If there are, only the first will be displayed. + """ + schema: [SchemaContractInput!] + + """ + The data quality portion of the contract. If not provided, this will be set to none. + """ + dataQuality: [DataQualityContractInput!] + + """ + The state of the data contract. If not provided, it will be in ACTIVE mode by default. + """ + state: DataContractState + + """ + Optional ID of the contract you want to create. Only applicable if this is a create operation. If not provided, a random + id will be generated for you. + """ + id: String +} + +""" +Input required to create an Freshness contract +""" +input FreshnessContractInput { + """ + The assertion monitoring this part of the data contract. Assertion must be of type Freshness. + """ + assertionUrn: String! +} + +""" +Input required to create a schema contract +""" +input SchemaContractInput { + """ + The assertion monitoring this part of the data contract. Assertion must be of type Data Schema. + """ + assertionUrn: String! +} + +""" +Input required to create a data quality contract +""" +input DataQualityContractInput { + """ + The assertion monitoring this part of the data contract. Assertion must be of type Dataset, Volume, Field / Column, or Custom SQL. + """ + assertionUrn: String! +} + diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index de030f77b0b01..246ace2fc0f5f 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -1158,6 +1158,11 @@ enum EntityType { """ ROLE + """ + A data contract + """ + DATA_CONTRACT + """" An structured property on entities """ @@ -4134,6 +4139,16 @@ type CorpUserEditableProperties { Email address for the user """ email: String + + """ + User persona, if present + """ + persona: DataHubPersona + + """ + Platforms commonly used by the user, if present. + """ + platforms: [DataPlatform!] } """ @@ -4184,6 +4199,16 @@ input CorpUserUpdateInput { Email address for the user """ email: String + + """ + The platforms that the user frequently works with + """ + platformUrns: [String!] + + """ + The user's persona urn" + """ + personaUrn: String } """ @@ -7323,6 +7348,11 @@ type AssertionInfo { An optional human-readable description of the assertion """ description: String + + """ + URL where assertion details are available + """ + externalUrl: String } """ @@ -7399,6 +7429,11 @@ type AssertionRunEvent implements TimeSeriesAspect { """ timestampMillis: Long! + """ + The time at which the run event was last observed by the DataHub system - ie, when it was reported by external systems + """ + lastObservedMillis: Long + """ Urn of assertion which is evaluated """ @@ -7415,7 +7450,7 @@ type AssertionRunEvent implements TimeSeriesAspect { runId: String! """ - The status of the assertion run as per this timeseries event. + The status of the assertion run as per this timeseries event """ status: AssertionRunStatus! @@ -7480,6 +7515,75 @@ type AssertionResult { """ nativeResults: [StringMapEntry!] + """ + Error details, if type is ERROR + """ + error: AssertionResultError +} + +""" +An error encountered when evaluating an AssertionResult +""" +type AssertionResultError { + """ + The type of error encountered + """ + type: AssertionResultErrorType! + + """ + Additional metadata depending on the type of error + """ + properties: [StringMapEntry!] +} + +""" +The type of error encountered when evaluating an AssertionResult +""" +enum AssertionResultErrorType { + """ + Source is unreachable + """ + SOURCE_CONNECTION_ERROR + + """ + Source query failed to execute + """ + SOURCE_QUERY_FAILED + + """ + Invalid parameters were detected + """ + INVALID_PARAMETERS + + """ + Insufficient data to evaluate assertion + """ + INSUFFICIENT_DATA + + """ + Event type not supported by the specified source + """ + INVALID_SOURCE_TYPE + + """ + Platform not supported + """ + UNSUPPORTED_PLATFORM + + """ + Error while executing a custom SQL assertion + """ + CUSTOM_SQL_ERROR + + """ + Error while executing a field assertion + """ + FIELD_ASSERTION_ERROR + + """ + Unknown error + """ + UNKNOWN_ERROR } type BatchSpec { @@ -7508,6 +7612,11 @@ type BatchSpec { The result type of an assertion, success or failure. """ enum AssertionResultType { + """ + The assertion has not yet been fully evaluated. + """ + INIT + """ The assertion succeeded. """ @@ -7517,6 +7626,11 @@ enum AssertionResultType { The assertion failed. """ FAILURE + + """ + The assertion errored. + """ + ERROR } """ @@ -7643,6 +7757,16 @@ enum AssertionStdOperator { """ EQUAL_TO + """ + Value being asserted is not equal to value + """ + NOT_EQUAL_TO + + """ + Value being asserted is null + """ + NULL + """ Value being asserted is not null """ @@ -7678,6 +7802,16 @@ enum AssertionStdOperator { """ NOT_IN + """ + Value being asserted is true + """ + IS_TRUE + + """ + Value being asserted is false + """ + IS_FALSE + """ Other """ @@ -7803,6 +7937,11 @@ enum AssertionType { A schema or structural assertion. """ DATA_SCHEMA + + """ + A custom assertion. + """ + CUSTOM } """ @@ -7824,6 +7963,11 @@ type AssertionRunEventsResult { """ succeeded: Int! + """ + The number of errored run events + """ + errored: Int! + """ The run events themselves """ @@ -8039,6 +8183,11 @@ type Deprecation { The user who will be credited for modifying this deprecation content """ actor: String + + """ + The hydrated user who will be credited for modifying this deprecation content + """ + actorEntity: Entity } """ @@ -12018,6 +12167,7 @@ input CreateDataProductPropertiesInput { description: String } + """ Input properties required for update a DataProduct """ @@ -12183,6 +12333,16 @@ input UpdateOwnershipTypeInput { description: String } +""" +A standardized type of a user +""" +type DataHubPersona { + """ + The urn of the persona type + """ + urn: String! +} + """ Describes a generic filter on a dataset """ @@ -12541,3 +12701,18 @@ type ListBusinessAttributesResult { """ businessAttributes: [BusinessAttribute!]! } + +""" +A cron schedule +""" +type CronSchedule { + """ + A cron-formatted execution interval, as a cron string, e.g. 1 * * * * + """ + cron: String! + + """ + Timezone in which the cron interval applies, e.g. America/Los_Angeles + """ + timezone: String! +} diff --git a/datahub-graphql-core/src/main/resources/forms.graphql b/datahub-graphql-core/src/main/resources/forms.graphql index f5e5fa74e3dc9..4a4e270509596 100644 --- a/datahub-graphql-core/src/main/resources/forms.graphql +++ b/datahub-graphql-core/src/main/resources/forms.graphql @@ -3,6 +3,21 @@ extend type Mutation { Remove a form from a given list of entities. """ batchRemoveForm(input: BatchRemoveFormInput!): Boolean! + + """ + Create a new form based on the input + """ + createForm(input: CreateFormInput!): Form! + + """ + Delete a given form + """ + deleteForm(input: DeleteFormInput!): Boolean! + + """ + Update an existing form based on the input + """ + updateForm(input: UpdateFormInput!): Form! } """ @@ -398,3 +413,184 @@ input BatchRemoveFormInput { """ entityUrns: [String!]! } + +""" +Input for batch removing a form from different entities +""" +input CreateFormInput { + """ + Advanced: Optionally provide an ID to create a form urn from + """ + id: String + + """ + The name of the form being created + """ + name: String! + + """ + The optional description of the form being created + """ + description: String + + """ + The type of this form, whether it's verification or completion. Default is completion. + """ + type: FormType + + """ + The type of this form, whether it's verification or completion. Default is completion. + """ + prompts: [CreatePromptInput!] + + """ + Information on how this form should be assigned to users/groups + """ + actors: FormActorAssignmentInput +} + +""" +Input for creating form prompts +""" +input CreatePromptInput { + """ + Advanced: Optionally provide an ID to this prompt. All prompt IDs must be globally unique. + """ + id: String + + """ + The title of the prompt + """ + title: String! + + """ + The optional description of the prompt + """ + description: String + + """ + The type of the prompt. + """ + type: FormPromptType! + + """ + The params required if this prompt type is STRUCTURED_PROPERTY or FIELDS_STRUCTURED_PROPERTY + """ + structuredPropertyParams: StructuredPropertyParamsInput + + """ + Whether this prompt will be required or not. Default is false. + """ + required: Boolean + +} + +""" +Input for assigning a form to actors +""" +input FormActorAssignmentInput { + """ + Whether this form will be applied to owners of associated entities or not. Default is true. + """ + owners: Boolean + + """ + The optional list of user urns to assign this form to + """ + users: [String!] + + """ + The optional list of group urns to assign this form to + """ + groups: [String!] +} + +""" +Input for a structured property type prompt +""" +input StructuredPropertyParamsInput { + """ + The urn of the structured property for a given form prompt + """ + urn: String! +} + +""" +Input for updating a form +""" +input UpdateFormInput { + """ + The urn of the form being updated + """ + urn: String! + + """ + The new name of the form + """ + name: String + + """ + The new description of the form + """ + description: String + + """ + The new type of the form + """ + type: FormType + + """ + The new prompts being added to this form + """ + promptsToAdd: [CreatePromptInput!] + + """ + The IDs of the prompts to remove from this form + """ + promptsToRemove: [String!] + + """ + Information on how this form should be assigned to users/groups + """ + actors: FormActorAssignmentUpdateInput +} + +""" +Update input for assigning a form to actors +""" +input FormActorAssignmentUpdateInput { + """ + Whether this form will be applied to owners of associated entities or not. Default is true. + """ + owners: Boolean + + """ + The optional list of user urns to assign this form to + """ + usersToAdd: [String!] + + """ + The users being removed from being assigned to this form + """ + usersToRemove: [String!] + + """ + The optional list of group urns to assign this form to + """ + groupsToAdd: [String!] + + """ + The groups being removed from being assigned to this form + """ + groupsToRemove: [String!] +} + +""" +Input for deleting a form +""" +input DeleteFormInput { + """ + The urn of the form that is being deleted + """ + urn: String! +} diff --git a/datahub-graphql-core/src/main/resources/incident.graphql b/datahub-graphql-core/src/main/resources/incident.graphql index f7060b3ae8f67..c2938543ed949 100644 --- a/datahub-graphql-core/src/main/resources/incident.graphql +++ b/datahub-graphql-core/src/main/resources/incident.graphql @@ -136,6 +136,36 @@ enum IncidentState { A specific type of incident """ enum IncidentType { + """ + A Freshness Assertion has failed, triggering the incident. + Raised on assets where assertions are configured to generate incidents. + """ + FRESHNESS + + """ + A Volume Assertion has failed, triggering the incident. + Raised on assets where assertions are configured to generate incidents. + """ + VOLUME + + """ + A Field Assertion has failed, triggering the incident. + Raised on assets where assertions are configured to generate incidents. + """ + FIELD + + """ + A SQL Assertion has failed, triggering the incident. + Raised on assets where assertions are configured to generate incidents. + """ + SQL + + """ + A Schema has failed, triggering the incident. + Raised on assets where assertions are configured to generate incidents. + """ + DATA_SCHEMA + """ An operational incident, e.g. failure to materialize a dataset, or failure to execute a task / pipeline. """ @@ -174,6 +204,11 @@ enum IncidentSourceType { The incident was created manually, from either the API or the UI. """ MANUAL + + """ + An assertion has failed, triggering the incident. + """ + ASSERTION_FAILURE } """ diff --git a/datahub-graphql-core/src/main/resources/properties.graphql b/datahub-graphql-core/src/main/resources/properties.graphql index 120154e930d59..dfe8468645681 100644 --- a/datahub-graphql-core/src/main/resources/properties.graphql +++ b/datahub-graphql-core/src/main/resources/properties.graphql @@ -3,6 +3,21 @@ extend type Mutation { Upsert structured properties onto a given asset """ upsertStructuredProperties(input: UpsertStructuredPropertiesInput!): StructuredProperties! + + """ + Upsert structured properties onto a given asset + """ + removeStructuredProperties(input: RemoveStructuredPropertiesInput!): StructuredProperties! + + """ + Create a new structured property + """ + createStructuredProperty(input: CreateStructuredPropertyInput!): StructuredPropertyEntity! + + """ + Update an existing structured property + """ + updateStructuredProperty(input: UpdateStructuredPropertyInput!): StructuredPropertyEntity! } """ @@ -184,6 +199,21 @@ input UpsertStructuredPropertiesInput { structuredPropertyInputParams: [StructuredPropertyInputParams!]! } +""" +Input for removing structured properties on a given asset +""" +input RemoveStructuredPropertiesInput { + """ + The urn of the asset that we are removing properties from + """ + assetUrn: String! + + """ + The list of structured properties you want to remove from this asset + """ + structuredPropertyUrns: [String!]! +} + """ A data type registered in DataHub """ @@ -268,3 +298,152 @@ type DataTypeInfo { """ description: String } + +""" +Input for creating a new structured property entity +""" +input CreateStructuredPropertyInput { + """ + (Advanced) An optional unique ID to use when creating the urn of this entity + """ + id: String + + """ + The unique fully qualified name of this structured property, dot delimited. + """ + qualifiedName: String! + + """ + The optional display name for this property + """ + displayName: String + + """ + The optional description for this property + """ + description: String + + """ + Whether the property will be mutable once it is applied or not. Default is false. + """ + immutable: Boolean + + """ + The urn of the value type that this structured property accepts. + For example: urn:li:dataType:datahub.string or urn:li:dataType:datahub.date + """ + valueType: String! + + """ + The optional input for specifying specific entity types as values + """ + typeQualifier: TypeQualifierInput + + """ + The optional input for specifying a list of allowed values + """ + allowedValues: [AllowedValueInput!] + + """ + The optional input for specifying if one or multiple values can be applied. + Default is one value (single cardinality) + """ + cardinality: PropertyCardinality + + """ + The list of entity types that this property can be applied to. + For example: ["urn:li:entityType:datahub.dataset"] + """ + entityTypes: [String!]! +} + +""" +Input for specifying specific entity types as values +""" +input TypeQualifierInput { + """ + The list of allowed entity types as urns (ie. ["urn:li:entityType:datahub.corpuser"]) + """ + allowedTypes: [String!] +} + +""" +An input entry for an allowed value for a structured property +""" +input AllowedValueInput { + """ + The allowed string value if the value is of type string + Either this or numberValue is required. + """ + stringValue: String + + """ + The allowed number value if the value is of type number. + Either this or stringValue is required. + """ + numberValue: Float + + """ + The description of this allowed value + """ + description: String +} + +""" +Input for updating an existing structured property entity +""" +input UpdateStructuredPropertyInput { + """ + The urn of the structured property being updated + """ + urn: String! + + """ + The optional display name for this property + """ + displayName: String + + """ + The optional description for this property + """ + description: String + + """ + Whether the property will be mutable once it is applied or not. Default is false. + """ + immutable: Boolean + + """ + The optional input for specifying specific entity types as values + """ + typeQualifier: UpdateTypeQualifierInput + + """ + Append to the list of allowed values for this property. + For backwards compatibility, this is append only. + """ + newAllowedValues: [AllowedValueInput!] + + """ + Set to true if you want to change the cardinality of this structured property + to multiple. Cannot change from multiple to single for backwards compatibility reasons. + """ + setCardinalityAsMultiple: Boolean + + """ + Append to the list of entity types that this property can be applied to. + For backwards compatibility, this is append only. + """ + newEntityTypes: [String!] +} + +""" +Input for updating specifying specific entity types as values +""" +input UpdateTypeQualifierInput { + """ + Append to the list of allowed entity types as urns for this property (ie. ["urn:li:entityType:datahub.corpuser"]) + For backwards compatibility, this is append only. + """ + newAllowedTypes: [String!] +} diff --git a/datahub-graphql-core/src/main/resources/search.graphql b/datahub-graphql-core/src/main/resources/search.graphql index 499ac3a0860d4..c7b5e61e9831c 100644 --- a/datahub-graphql-core/src/main/resources/search.graphql +++ b/datahub-graphql-core/src/main/resources/search.graphql @@ -747,6 +747,11 @@ type SearchAcrossLineageResult { """ explored: Boolean! + """ + Indicates this destination node has additional unexplored child relationships + """ + truncatedChildren: Boolean! + """ Whether this relationship was ignored as a hop """ diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java index 57d85e5b204c2..f98284e92ede5 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql.resolvers; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static org.mockito.Mockito.mock; import static org.testng.AssertJUnit.assertEquals; import com.google.common.collect.ImmutableList; @@ -11,6 +12,7 @@ import com.linkedin.datahub.graphql.TestUtils; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.FilterOperator; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; @@ -28,7 +30,7 @@ public class ResolverUtilsTest { @Test public void testCriterionFromFilter() throws Exception { - final DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + final DataFetchingEnvironment mockEnv = mock(DataFetchingEnvironment.class); final QueryContext mockAllowContext = TestUtils.getMockAllowContext(); Mockito.when(mockEnv.getContext()).thenReturn(mockAllowContext); @@ -40,7 +42,8 @@ public void testCriterionFromFilter() throws Exception { null, ImmutableList.of("urn:li:tag:abc", "urn:li:tag:def"), false, - FilterOperator.EQUAL)); + FilterOperator.EQUAL), + mock(AspectRetriever.class)); assertEquals( valuesCriterion, new Criterion() @@ -53,7 +56,8 @@ public void testCriterionFromFilter() throws Exception { // this is the legacy pathway Criterion valueCriterion = criterionFromFilter( - new FacetFilterInput("tags", "urn:li:tag:abc", null, true, FilterOperator.EQUAL)); + new FacetFilterInput("tags", "urn:li:tag:abc", null, true, FilterOperator.EQUAL), + mock(AspectRetriever.class)); assertEquals( valueCriterion, new Criterion() @@ -66,7 +70,9 @@ public void testCriterionFromFilter() throws Exception { // check that both being null doesn't cause a NPE. this should never happen except via API // interaction Criterion doubleNullCriterion = - criterionFromFilter(new FacetFilterInput("tags", null, null, true, FilterOperator.EQUAL)); + criterionFromFilter( + new FacetFilterInput("tags", null, null, true, FilterOperator.EQUAL), + mock(AspectRetriever.class)); assertEquals( doubleNullCriterion, new Criterion() diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolverTest.java index 7323a62d94bfe..f6e7e7267a060 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolverTest.java @@ -18,7 +18,9 @@ import com.linkedin.metadata.aspect.EnvelopedAspect; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.SystemMetadata; import graphql.schema.DataFetchingEnvironment; +import io.datahubproject.metadata.context.OperationContext; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -55,16 +57,21 @@ public void testGetSuccess() throws Exception { Mockito.eq(5), Mockito.eq( AssertionRunEventResolver.buildFilter( - null, AssertionRunStatus.COMPLETE.toString())))) + null, AssertionRunStatus.COMPLETE.toString(), null)))) .thenReturn( ImmutableList.of( - new EnvelopedAspect().setAspect(GenericRecordUtils.serializeAspect(gmsRunEvent)))); + new EnvelopedAspect() + .setAspect(GenericRecordUtils.serializeAspect(gmsRunEvent)) + .setSystemMetadata(new SystemMetadata().setLastObserved(12L)))); AssertionRunEventResolver resolver = new AssertionRunEventResolver(mockClient); // Execute resolver QueryContext mockContext = Mockito.mock(QueryContext.class); Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + Mockito.when(mockContext.getOperationContext()) + .thenReturn(Mockito.mock(OperationContext.class)); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); Mockito.when(mockEnv.getArgumentOrDefault(Mockito.eq("status"), Mockito.eq(null))) @@ -97,6 +104,7 @@ public void testGetSuccess() throws Exception { assertEquals(result.getTotal(), 1); assertEquals(result.getFailed(), 0); assertEquals(result.getSucceeded(), 1); + assertEquals(result.getErrored(), 0); com.linkedin.datahub.graphql.generated.AssertionRunEvent graphqlRunEvent = resolver.get(mockEnv).get().getRunEvents().get(0); @@ -107,6 +115,7 @@ public void testGetSuccess() throws Exception { graphqlRunEvent.getStatus(), com.linkedin.datahub.graphql.generated.AssertionRunStatus.COMPLETE); assertEquals((float) graphqlRunEvent.getTimestampMillis(), 12L); + assertEquals((float) graphqlRunEvent.getLastObservedMillis(), 12L); assertEquals((float) graphqlRunEvent.getResult().getActualAggValue(), 10); assertEquals((long) graphqlRunEvent.getResult().getMissingCount(), 0L); assertEquals((long) graphqlRunEvent.getResult().getRowCount(), 1L); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/ReportAssertionResultResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/ReportAssertionResultResolverTest.java new file mode 100644 index 0000000000000..cf3c833cbba23 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/ReportAssertionResultResolverTest.java @@ -0,0 +1,160 @@ +package com.linkedin.datahub.graphql.resolvers.assertion; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.mockito.ArgumentMatchers.*; +import static org.testng.Assert.*; + +import com.google.common.collect.ImmutableList; +import com.linkedin.assertion.AssertionResult; +import com.linkedin.assertion.AssertionResultError; +import com.linkedin.assertion.AssertionRunEvent; +import com.linkedin.assertion.AssertionRunStatus; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringMap; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.AssertionResultErrorInput; +import com.linkedin.datahub.graphql.generated.AssertionResultErrorType; +import com.linkedin.datahub.graphql.generated.AssertionResultInput; +import com.linkedin.datahub.graphql.generated.AssertionResultType; +import com.linkedin.datahub.graphql.generated.StringMapEntryInput; +import com.linkedin.metadata.service.AssertionService; +import graphql.schema.DataFetchingEnvironment; +import io.datahubproject.metadata.context.OperationContext; +import java.util.Map; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class ReportAssertionResultResolverTest { + + private static final Urn TEST_DATASET_URN = + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,name,PROD)"); + + private static final Urn TEST_ASSERTION_URN = UrnUtils.getUrn("urn:li:assertion:test"); + + private static final String customAssertionUrl = "https://dq-platform-native-url"; + + private static final AssertionResultInput TEST_INPUT = + new AssertionResultInput( + 0L, + AssertionResultType.ERROR, + ImmutableList.of(new StringMapEntryInput("prop1", "value1")), + customAssertionUrl, + new AssertionResultErrorInput( + AssertionResultErrorType.UNKNOWN_ERROR, "an unknown error occurred")); + + ; + + private static final AssertionRunEvent TEST_ASSERTION_RUN_EVENT = + new AssertionRunEvent() + .setAssertionUrn(TEST_ASSERTION_URN) + .setAsserteeUrn(TEST_DATASET_URN) + .setTimestampMillis(0L) + .setRunId("0") + .setStatus(AssertionRunStatus.COMPLETE) + .setResult( + new AssertionResult() + .setType(com.linkedin.assertion.AssertionResultType.ERROR) + .setError( + new AssertionResultError() + .setType(com.linkedin.assertion.AssertionResultErrorType.UNKNOWN_ERROR) + .setProperties( + new StringMap(Map.of("message", "an unknown error occurred")))) + .setExternalUrl(customAssertionUrl) + .setNativeResults(new StringMap(Map.of("prop1", "value1")))); + + @Test + public void testGetSuccessReportAssertionRunEvent() throws Exception { + // Update resolver + AssertionService mockedService = Mockito.mock(AssertionService.class); + ReportAssertionResultResolver resolver = new ReportAssertionResultResolver(mockedService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_ASSERTION_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("result"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Mockito.when( + mockedService.getEntityUrnForAssertion( + any(OperationContext.class), Mockito.eq(TEST_ASSERTION_URN))) + .thenReturn(TEST_DATASET_URN); + + resolver.get(mockEnv).get(); + + // Validate that we created the assertion + Mockito.verify(mockedService, Mockito.times(1)) + .addAssertionRunEvent( + any(OperationContext.class), + Mockito.eq(TEST_ASSERTION_URN), + Mockito.eq(TEST_DATASET_URN), + Mockito.eq(TEST_ASSERTION_RUN_EVENT.getTimestampMillis()), + Mockito.eq(TEST_ASSERTION_RUN_EVENT.getResult())); + } + + @Test + public void testGetUpdateAssertionUnauthorized() throws Exception { + // Update resolver + AssertionService mockedService = Mockito.mock(AssertionService.class); + ReportAssertionResultResolver resolver = new ReportAssertionResultResolver(mockedService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_ASSERTION_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("result"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Mockito.when( + mockedService.getEntityUrnForAssertion( + any(OperationContext.class), Mockito.eq(TEST_ASSERTION_URN))) + .thenReturn(TEST_DATASET_URN); + + CompletionException e = + expectThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + assert e.getMessage() + .contains( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + + // Validate that we created the assertion + Mockito.verify(mockedService, Mockito.times(0)) + .addAssertionRunEvent( + any(OperationContext.class), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any()); + } + + @Test + public void testGetAssertionServiceException() { + // Update resolver + AssertionService mockService = Mockito.mock(AssertionService.class); + + Mockito.when( + mockService.getEntityUrnForAssertion( + any(OperationContext.class), Mockito.eq(TEST_ASSERTION_URN))) + .thenReturn(TEST_DATASET_URN); + Mockito.doThrow(RuntimeException.class) + .when(mockService) + .addAssertionRunEvent( + any(OperationContext.class), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any()); + + ReportAssertionResultResolver resolver = new ReportAssertionResultResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_ASSERTION_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("result"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/UpsertCustomAssertionResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/UpsertCustomAssertionResolverTest.java new file mode 100644 index 0000000000000..2ac6335ba9fea --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/assertion/UpsertCustomAssertionResolverTest.java @@ -0,0 +1,345 @@ +package com.linkedin.datahub.graphql.resolvers.assertion; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.mockito.ArgumentMatchers.*; +import static org.testng.Assert.*; + +import com.google.common.collect.ImmutableMap; +import com.linkedin.assertion.AssertionInfo; +import com.linkedin.assertion.AssertionSource; +import com.linkedin.assertion.AssertionSourceType; +import com.linkedin.assertion.AssertionType; +import com.linkedin.assertion.CustomAssertionInfo; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.DataPlatformInstance; +import com.linkedin.common.url.Url; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.Assertion; +import com.linkedin.datahub.graphql.generated.PlatformInput; +import com.linkedin.datahub.graphql.generated.UpsertCustomAssertionInput; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.service.AssertionService; +import graphql.schema.DataFetchingEnvironment; +import io.datahubproject.metadata.context.OperationContext; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class UpsertCustomAssertionResolverTest { + + private static final Urn TEST_DATASET_URN = + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,name,PROD)"); + + private static final String TEST_INVALID_DATASET_URN = "dataset.name"; + + private static final Urn TEST_FIELD_URN = + UrnUtils.getUrn( + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,name,PROD),field1)"); + private static final Urn TEST_ASSERTION_URN = UrnUtils.getUrn("urn:li:assertion:test"); + + private static final String TEST_INVALID_ASSERTION_URN = "test"; + private static final Urn TEST_ACTOR_URN = UrnUtils.getUrn("urn:li:actor:test"); + + private static final Urn TEST_PLATFORM_URN = UrnUtils.getUrn("urn:li:dataPlatform:DQplatform"); + + private static final String customAssertionType = "My custom category"; + private static final String customAssertionDescription = "Description of custom assertion"; + private static final String customAssertionUrl = "https://dq-platform-native-url"; + + private static final String customAssertionLogic = "custom script of assertion"; + + private static final UpsertCustomAssertionInput TEST_INPUT = + new UpsertCustomAssertionInput( + TEST_DATASET_URN.toString(), + customAssertionType, + customAssertionDescription, + "field1", + new PlatformInput(null, "DQplatform"), + customAssertionUrl, + customAssertionLogic); + + private static final UpsertCustomAssertionInput TEST_INPUT_MISSING_PLATFORM = + new UpsertCustomAssertionInput( + TEST_DATASET_URN.toString(), + customAssertionType, + customAssertionDescription, + "field1", + new PlatformInput(null, null), + customAssertionUrl, + customAssertionLogic); + + private static final UpsertCustomAssertionInput TEST_INPUT_INVALID_ENTITY_URN = + new UpsertCustomAssertionInput( + TEST_INVALID_DATASET_URN, + customAssertionType, + customAssertionDescription, + "field1", + new PlatformInput(null, "DQplatform"), + customAssertionUrl, + customAssertionLogic); + + private static final AssertionInfo TEST_ASSERTION_INFO = + new AssertionInfo() + .setType(AssertionType.CUSTOM) + .setDescription(customAssertionDescription) + .setExternalUrl(new Url(customAssertionUrl)) + .setSource( + new AssertionSource() + .setType(AssertionSourceType.EXTERNAL) + .setCreated( + new AuditStamp() + .setTime(System.currentTimeMillis()) + .setActor(TEST_ACTOR_URN))) + .setCustomAssertion( + new CustomAssertionInfo() + .setEntity(TEST_DATASET_URN) + .setType(customAssertionType) + .setField(TEST_FIELD_URN) + .setLogic(customAssertionLogic)); + + private static final DataPlatformInstance TEST_DATA_PLATFORM_INSTANCE = + new DataPlatformInstance().setPlatform(TEST_PLATFORM_URN); + + @Test + public void testGetSuccessCreateAssertion() throws Exception { + // Update resolver + AssertionService mockedService = Mockito.mock(AssertionService.class); + UpsertCustomAssertionResolver resolver = new UpsertCustomAssertionResolver(mockedService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(null); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Mockito.when(mockedService.generateAssertionUrn()).thenReturn(TEST_ASSERTION_URN); + Mockito.when( + mockedService.getAssertionEntityResponse( + any(OperationContext.class), Mockito.eq(TEST_ASSERTION_URN))) + .thenReturn( + new EntityResponse() + .setAspects( + new EnvelopedAspectMap( + ImmutableMap.of( + Constants.ASSERTION_INFO_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(TEST_ASSERTION_INFO.data())), + Constants.DATA_PLATFORM_INSTANCE_ASPECT_NAME, + new EnvelopedAspect() + .setValue(new Aspect(TEST_DATA_PLATFORM_INSTANCE.data()))))) + .setEntityName(Constants.ASSERTION_ENTITY_NAME) + .setUrn(TEST_ASSERTION_URN)); + + Assertion assertion = resolver.get(mockEnv).get(); + + // Don't validate each field since we have mapper tests already. + assertNotNull(assertion); + assertEquals(assertion.getUrn(), TEST_ASSERTION_URN.toString()); + + // Validate that we created the assertion + Mockito.verify(mockedService, Mockito.times(1)) + .upsertCustomAssertion( + any(OperationContext.class), + Mockito.eq(TEST_ASSERTION_URN), + Mockito.eq(TEST_ASSERTION_INFO.getCustomAssertion().getEntity()), + Mockito.eq(TEST_ASSERTION_INFO.getDescription()), + Mockito.eq(TEST_ASSERTION_INFO.getExternalUrl().toString()), + Mockito.eq(TEST_DATA_PLATFORM_INSTANCE), + Mockito.eq(TEST_ASSERTION_INFO.getCustomAssertion())); + } + + @Test + public void testGetSuccessUpdateAssertion() throws Exception { + // Update resolver + AssertionService mockedService = Mockito.mock(AssertionService.class); + UpsertCustomAssertionResolver resolver = new UpsertCustomAssertionResolver(mockedService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_ASSERTION_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Mockito.when( + mockedService.getAssertionEntityResponse( + any(OperationContext.class), Mockito.eq(TEST_ASSERTION_URN))) + .thenReturn( + new EntityResponse() + .setAspects( + new EnvelopedAspectMap( + ImmutableMap.of( + Constants.ASSERTION_INFO_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(TEST_ASSERTION_INFO.data())), + Constants.DATA_PLATFORM_INSTANCE_ASPECT_NAME, + new EnvelopedAspect() + .setValue(new Aspect(TEST_DATA_PLATFORM_INSTANCE.data()))))) + .setEntityName(Constants.ASSERTION_ENTITY_NAME) + .setUrn(TEST_ASSERTION_URN)); + + Assertion assertion = resolver.get(mockEnv).get(); + + // Don't validate each field since we have mapper tests already. + assertNotNull(assertion); + assertEquals(assertion.getUrn(), TEST_ASSERTION_URN.toString()); + + // Validate that we created the assertion + Mockito.verify(mockedService, Mockito.times(1)) + .upsertCustomAssertion( + any(OperationContext.class), + Mockito.eq(TEST_ASSERTION_URN), + Mockito.eq(TEST_ASSERTION_INFO.getCustomAssertion().getEntity()), + Mockito.eq(TEST_ASSERTION_INFO.getDescription()), + Mockito.eq(TEST_ASSERTION_INFO.getExternalUrl().toString()), + Mockito.eq(TEST_DATA_PLATFORM_INSTANCE), + Mockito.eq(TEST_ASSERTION_INFO.getCustomAssertion())); + } + + @Test + public void testGetUpdateAssertionUnauthorized() throws Exception { + // Update resolver + AssertionService mockedService = Mockito.mock(AssertionService.class); + UpsertCustomAssertionResolver resolver = new UpsertCustomAssertionResolver(mockedService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_ASSERTION_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + CompletionException e = + expectThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + assert e.getMessage() + .contains( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + + Mockito.verify(mockedService, Mockito.times(0)) + .upsertCustomAssertion( + any(OperationContext.class), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any()); + } + + @Test + public void testGetUpsertAssertionMissingPlatformFailure() throws Exception { + // Update resolver + AssertionService mockedService = Mockito.mock(AssertionService.class); + UpsertCustomAssertionResolver resolver = new UpsertCustomAssertionResolver(mockedService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_ASSERTION_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT_MISSING_PLATFORM); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + CompletionException e = + expectThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + assert e.getMessage() + .contains( + "Failed to upsert Custom Assertion. Platform Name or Platform Urn must be specified."); + + Mockito.verify(mockedService, Mockito.times(0)) + .upsertCustomAssertion( + any(OperationContext.class), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any()); + } + + @Test + public void testGetUpsertAssertionInvalidAssertionUrn() throws Exception { + // Update resolver + AssertionService mockedService = Mockito.mock(AssertionService.class); + UpsertCustomAssertionResolver resolver = new UpsertCustomAssertionResolver(mockedService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_INVALID_ASSERTION_URN); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + RuntimeException e = expectThrows(RuntimeException.class, () -> resolver.get(mockEnv).join()); + assert e.getMessage().contains("invalid urn"); + + Mockito.verify(mockedService, Mockito.times(0)) + .upsertCustomAssertion( + any(OperationContext.class), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any()); + } + + @Test + public void testGetUpsertAssertionInvalidEntityUrn() throws Exception { + // Update resolver + AssertionService mockedService = Mockito.mock(AssertionService.class); + UpsertCustomAssertionResolver resolver = new UpsertCustomAssertionResolver(mockedService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_ASSERTION_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))) + .thenReturn(TEST_INPUT_INVALID_ENTITY_URN); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + RuntimeException e = expectThrows(RuntimeException.class, () -> resolver.get(mockEnv).join()); + assert e.getMessage().contains("invalid urn"); + + Mockito.verify(mockedService, Mockito.times(0)) + .upsertCustomAssertion( + any(OperationContext.class), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any()); + } + + @Test + public void testGetAssertionServiceException() { + // Update resolver + AssertionService mockService = Mockito.mock(AssertionService.class); + Mockito.doThrow(RuntimeException.class) + .when(mockService) + .upsertCustomAssertion( + any(OperationContext.class), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any()); + + UpsertCustomAssertionResolver resolver = new UpsertCustomAssertionResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_ASSERTION_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java index ad30e48d8361b..6c876226a45e6 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java @@ -46,7 +46,7 @@ public void testGetSuccess() throws Exception { any(), Mockito.eq(Constants.ACCESS_TOKEN_ENTITY_NAME), Mockito.eq(""), - Mockito.eq(buildFilter(filters, Collections.emptyList())), + Mockito.eq(buildFilter(filters, Collections.emptyList(), null)), Mockito.any(SortCriterion.class), Mockito.eq(input.getStart()), Mockito.eq(input.getCount()))) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java index 4897d0819b59f..9cf7e62e65e25 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java @@ -2,6 +2,7 @@ import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import com.google.common.collect.ImmutableList; import com.linkedin.common.AuditStamp; @@ -17,6 +18,7 @@ import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.resolvers.chart.BrowseV2Resolver; import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.browse.BrowseResultGroupV2; import com.linkedin.metadata.browse.BrowseResultGroupV2Array; import com.linkedin.metadata.browse.BrowseResultMetadata; @@ -100,7 +102,7 @@ public static void testBrowseV2SuccessWithQueryAndFilter() throws Exception { facetFilterInput.setValues(ImmutableList.of("urn:li:corpuser:test")); andFilterInput.setAnd(ImmutableList.of(facetFilterInput)); orFilters.add(andFilterInput); - Filter filter = ResolverUtils.buildFilter(null, orFilters); + Filter filter = ResolverUtils.buildFilter(null, orFilters, mock(AspectRetriever.class)); EntityClient mockClient = initMockEntityClient( diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/DataContractUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/DataContractUtilsTest.java new file mode 100644 index 0000000000000..18ede7c306e42 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/DataContractUtilsTest.java @@ -0,0 +1,63 @@ +package com.linkedin.datahub.graphql.resolvers.datacontract; + +import static org.mockito.Mockito.mock; + +import com.datahub.authentication.Actor; +import com.datahub.authentication.ActorType; +import com.datahub.authentication.Authentication; +import com.datahub.authorization.AuthorizationRequest; +import com.datahub.authorization.AuthorizationResult; +import com.datahub.authorization.EntitySpec; +import com.datahub.plugins.auth.authorization.Authorizer; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import graphql.Assert; +import io.datahubproject.metadata.context.OperationContext; +import io.datahubproject.test.metadata.context.TestOperationContexts; +import java.util.Optional; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class DataContractUtilsTest { + + @Test + public void testCanEditDataContract() { + Urn testUrn = UrnUtils.getUrn("urn:li:dataContract:test"); + boolean result = + DataContractUtils.canEditDataContract( + new QueryContext() { + @Override + public boolean isAuthenticated() { + return true; + } + + @Override + public Authentication getAuthentication() { + Authentication auth = new Authentication(new Actor(ActorType.USER, "test"), "TEST"); + return auth; + } + + @Override + public Authorizer getAuthorizer() { + Authorizer authorizer = mock(Authorizer.class); + Mockito.when(authorizer.authorize(Mockito.any(AuthorizationRequest.class))) + .thenReturn( + new AuthorizationResult( + new AuthorizationRequest( + "TEST", "test", Optional.of(new EntitySpec("dataset", "test"))), + AuthorizationResult.Type.ALLOW, + "TEST")); + return authorizer; + } + + @Override + public OperationContext getOperationContext() { + return TestOperationContexts.userContextNoSearchAuthorization( + getAuthorizer(), getAuthentication()); + } + }, + testUrn); + Assert.assertTrue(result); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/EntityDataContractResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/EntityDataContractResolverTest.java new file mode 100644 index 0000000000000..8b757a24d6566 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/EntityDataContractResolverTest.java @@ -0,0 +1,206 @@ +package com.linkedin.datahub.graphql.resolvers.datacontract; + +import static com.linkedin.datahub.graphql.resolvers.datacontract.EntityDataContractResolver.*; +import static org.mockito.ArgumentMatchers.nullable; +import static org.testng.Assert.*; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.EntityRelationship; +import com.linkedin.common.EntityRelationshipArray; +import com.linkedin.common.EntityRelationships; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datacontract.DataContractProperties; +import com.linkedin.datacontract.DataContractState; +import com.linkedin.datacontract.DataContractStatus; +import com.linkedin.datacontract.DataQualityContract; +import com.linkedin.datacontract.DataQualityContractArray; +import com.linkedin.datacontract.FreshnessContract; +import com.linkedin.datacontract.FreshnessContractArray; +import com.linkedin.datacontract.SchemaContract; +import com.linkedin.datacontract.SchemaContractArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DataContract; +import com.linkedin.datahub.graphql.generated.Dataset; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.graph.GraphClient; +import com.linkedin.metadata.key.DataContractKey; +import com.linkedin.metadata.query.filter.RelationshipDirection; +import graphql.schema.DataFetchingEnvironment; +import io.datahubproject.metadata.context.OperationContext; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class EntityDataContractResolverTest { + + private static final Urn TEST_DATASET_URN = + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:snowflake,test,PROD)"); + private static final Urn TEST_DATA_CONTRACT_URN = UrnUtils.getUrn("urn:li:dataContract:test"); + private static final Urn TEST_QUALITY_ASSERTION_URN = UrnUtils.getUrn("urn:li:assertion:quality"); + private static final Urn TEST_FRESHNESS_ASSERTION_URN = + UrnUtils.getUrn("urn:li:assertion:freshness"); + private static final Urn TEST_SCHEMA_ASSERTION_URN = UrnUtils.getUrn("urn:li:assertion:schema"); + + @Test + public void testGetSuccessOneContract() throws Exception { + GraphClient mockGraphClient = Mockito.mock(GraphClient.class); + EntityClient mockClient = Mockito.mock(EntityClient.class); + + Mockito.when( + mockGraphClient.getRelatedEntities( + Mockito.eq(TEST_DATASET_URN.toString()), + Mockito.eq(ImmutableList.of(CONTRACT_FOR_RELATIONSHIP)), + Mockito.eq(RelationshipDirection.INCOMING), + Mockito.eq(0), + Mockito.eq(1), + Mockito.anyString())) + .thenReturn( + new EntityRelationships() + .setTotal(1) + .setCount(1) + .setStart(0) + .setRelationships( + new EntityRelationshipArray( + ImmutableList.of( + new EntityRelationship() + .setType(CONTRACT_FOR_RELATIONSHIP) + .setEntity(TEST_DATA_CONTRACT_URN) + .setCreated( + new AuditStamp() + .setActor(UrnUtils.getUrn("urn:li:corpuser:test")) + .setTime(0L)))))); + + Map dataContractAspects = new HashMap<>(); + + // 1. Key Aspect + dataContractAspects.put( + Constants.DATA_CONTRACT_KEY_ASPECT_NAME, + new com.linkedin.entity.EnvelopedAspect() + .setValue(new Aspect(new DataContractKey().setId("test").data()))); + + // 2. Properties Aspect. + DataContractProperties expectedProperties = + new DataContractProperties() + .setEntity(TEST_DATASET_URN) + .setDataQuality( + new DataQualityContractArray( + ImmutableList.of( + new DataQualityContract().setAssertion(TEST_QUALITY_ASSERTION_URN)))) + .setFreshness( + new FreshnessContractArray( + ImmutableList.of( + new FreshnessContract().setAssertion(TEST_FRESHNESS_ASSERTION_URN)))) + .setSchema( + new SchemaContractArray( + ImmutableList.of( + new SchemaContract().setAssertion(TEST_SCHEMA_ASSERTION_URN)))); + + dataContractAspects.put( + Constants.DATA_CONTRACT_PROPERTIES_ASPECT_NAME, + new com.linkedin.entity.EnvelopedAspect().setValue(new Aspect(expectedProperties.data()))); + + // 3. Status Aspect + DataContractStatus expectedStatus = new DataContractStatus().setState(DataContractState.ACTIVE); + + dataContractAspects.put( + Constants.DATA_CONTRACT_STATUS_ASPECT_NAME, + new com.linkedin.entity.EnvelopedAspect().setValue(new Aspect(expectedStatus.data()))); + + Mockito.when( + mockClient.getV2( + nullable(OperationContext.class), + Mockito.eq(Constants.DATA_CONTRACT_ENTITY_NAME), + Mockito.eq(TEST_DATA_CONTRACT_URN), + Mockito.eq(null))) + .thenReturn( + new EntityResponse() + .setEntityName(Constants.DATA_CONTRACT_ENTITY_NAME) + .setUrn(TEST_DATA_CONTRACT_URN) + .setAspects(new EnvelopedAspectMap(dataContractAspects))); + + // Execute resolver + QueryContext mockContext = Mockito.mock(QueryContext.class); + Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + Mockito.when(mockContext.getActorUrn()).thenReturn("urn:li:corpuser:test"); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Dataset parentDataset = new Dataset(); + parentDataset.setUrn(TEST_DATASET_URN.toString()); + Mockito.when(mockEnv.getSource()).thenReturn(parentDataset); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + EntityDataContractResolver resolver = + new EntityDataContractResolver(mockClient, mockGraphClient); + DataContract result = resolver.get(mockEnv).get(); + + // Assert that the result we get matches the expectations. + assertEquals(result.getUrn(), TEST_DATA_CONTRACT_URN.toString()); + assertEquals(result.getType(), EntityType.DATA_CONTRACT); + + // Verify Properties + assertEquals(result.getProperties().getDataQuality().size(), 1); + assertEquals(result.getProperties().getFreshness().size(), 1); + assertEquals(result.getProperties().getSchema().size(), 1); + assertEquals( + result.getProperties().getDataQuality().get(0).getAssertion().getUrn(), + TEST_QUALITY_ASSERTION_URN.toString()); + assertEquals( + result.getProperties().getFreshness().get(0).getAssertion().getUrn(), + TEST_FRESHNESS_ASSERTION_URN.toString()); + assertEquals( + result.getProperties().getSchema().get(0).getAssertion().getUrn(), + TEST_SCHEMA_ASSERTION_URN.toString()); + + // Verify Status + assertEquals(result.getStatus().getState().toString(), expectedStatus.getState().toString()); + } + + @Test + public void testGetSuccessNoContracts() throws Exception { + GraphClient mockGraphClient = Mockito.mock(GraphClient.class); + EntityClient mockClient = Mockito.mock(EntityClient.class); + + Mockito.when( + mockGraphClient.getRelatedEntities( + Mockito.eq(TEST_DATASET_URN.toString()), + Mockito.eq(ImmutableList.of(CONTRACT_FOR_RELATIONSHIP)), + Mockito.eq(RelationshipDirection.INCOMING), + Mockito.eq(0), + Mockito.eq(1), + Mockito.anyString())) + .thenReturn( + new EntityRelationships() + .setTotal(0) + .setCount(0) + .setStart(0) + .setRelationships(new EntityRelationshipArray(Collections.emptyList()))); + + EntityDataContractResolver resolver = + new EntityDataContractResolver(mockClient, mockGraphClient); + + // Execute resolver + QueryContext mockContext = Mockito.mock(QueryContext.class); + Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + Mockito.when(mockContext.getActorUrn()).thenReturn("urn:li:corpuser:test"); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Dataset parentDataset = new Dataset(); + parentDataset.setUrn(TEST_DATASET_URN.toString()); + Mockito.when(mockEnv.getSource()).thenReturn(parentDataset); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + DataContract result = resolver.get(mockEnv).get(); + + assertNull(result); + Mockito.verifyNoMoreInteractions(mockClient); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/UpsertDataContractResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/UpsertDataContractResolverTest.java new file mode 100644 index 0000000000000..601fc56b25149 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/datacontract/UpsertDataContractResolverTest.java @@ -0,0 +1,379 @@ +package com.linkedin.datahub.graphql.resolvers.datacontract; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.datahub.graphql.resolvers.datacontract.EntityDataContractResolver.*; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.*; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.EntityRelationship; +import com.linkedin.common.EntityRelationshipArray; +import com.linkedin.common.EntityRelationships; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringMap; +import com.linkedin.datacontract.DataContractProperties; +import com.linkedin.datacontract.DataContractStatus; +import com.linkedin.datacontract.DataQualityContract; +import com.linkedin.datacontract.DataQualityContractArray; +import com.linkedin.datacontract.FreshnessContract; +import com.linkedin.datacontract.FreshnessContractArray; +import com.linkedin.datacontract.SchemaContract; +import com.linkedin.datacontract.SchemaContractArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DataContract; +import com.linkedin.datahub.graphql.generated.DataContractState; +import com.linkedin.datahub.graphql.generated.DataQualityContractInput; +import com.linkedin.datahub.graphql.generated.FreshnessContractInput; +import com.linkedin.datahub.graphql.generated.SchemaContractInput; +import com.linkedin.datahub.graphql.generated.UpsertDataContractInput; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.AspectType; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.graph.GraphClient; +import com.linkedin.metadata.key.DataContractKey; +import com.linkedin.metadata.query.filter.RelationshipDirection; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.mxe.SystemMetadata; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetchingEnvironment; +import io.datahubproject.metadata.context.OperationContext; +import java.util.Collections; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class UpsertDataContractResolverTest { + + private static final Urn TEST_CONTRACT_URN = UrnUtils.getUrn("urn:li:dataContract:test-id"); + private static final Urn TEST_DATASET_URN = + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:snowflake,test,PROD)"); + private static final Urn TEST_FRESHNESS_ASSERTION_URN = + UrnUtils.getUrn("urn:li:assertion:freshness"); + private static final Urn TEST_SCHEMA_ASSERTION_URN = UrnUtils.getUrn("urn:li:assertion:schema"); + private static final Urn TEST_QUALITY_ASSERTION_URN = UrnUtils.getUrn("urn:li:assertion:quality"); + + private static final UpsertDataContractInput TEST_CREATE_INPUT = + new UpsertDataContractInput( + TEST_DATASET_URN.toString(), + ImmutableList.of(new FreshnessContractInput(TEST_FRESHNESS_ASSERTION_URN.toString())), + ImmutableList.of(new SchemaContractInput(TEST_SCHEMA_ASSERTION_URN.toString())), + ImmutableList.of(new DataQualityContractInput(TEST_QUALITY_ASSERTION_URN.toString())), + DataContractState.PENDING, + "test-id"); + + private static final UpsertDataContractInput TEST_VALID_UPDATE_INPUT = + new UpsertDataContractInput( + TEST_DATASET_URN.toString(), + ImmutableList.of(new FreshnessContractInput(TEST_FRESHNESS_ASSERTION_URN.toString())), + ImmutableList.of(new SchemaContractInput(TEST_SCHEMA_ASSERTION_URN.toString())), + ImmutableList.of(new DataQualityContractInput(TEST_QUALITY_ASSERTION_URN.toString())), + DataContractState.ACTIVE, + null); + + private static final Urn TEST_ACTOR_URN = UrnUtils.getUrn("urn:li:corpuser:test"); + + @Test + public void testGetSuccessCreate() throws Exception { + + // Expected results + final DataContractKey key = new DataContractKey(); + key.setId("test-id"); + final Urn dataContractUrn = + EntityKeyUtils.convertEntityKeyToUrn(key, Constants.DATA_CONTRACT_ENTITY_NAME); + + final DataContractStatus status = new DataContractStatus(); + status.setState(com.linkedin.datacontract.DataContractState.PENDING); + + final DataContractProperties props = new DataContractProperties(); + props.setEntity(TEST_DATASET_URN); + props.setDataQuality( + new DataQualityContractArray( + ImmutableList.of(new DataQualityContract().setAssertion(TEST_QUALITY_ASSERTION_URN)))); + props.setFreshness( + new FreshnessContractArray( + ImmutableList.of(new FreshnessContract().setAssertion(TEST_FRESHNESS_ASSERTION_URN)))); + props.setSchema( + new SchemaContractArray( + ImmutableList.of(new SchemaContract().setAssertion(TEST_SCHEMA_ASSERTION_URN)))); + + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + GraphClient mockGraphClient = Mockito.mock(GraphClient.class); + initMockGraphClient(mockGraphClient, null); + initMockEntityClient(mockClient, null, props); // No existing contract + UpsertDataContractResolver resolver = + new UpsertDataContractResolver(mockClient, mockGraphClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_CREATE_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + DataContract result = resolver.get(mockEnv).get(); + + final MetadataChangeProposal propertiesProposal = new MetadataChangeProposal(); + propertiesProposal.setEntityUrn(dataContractUrn); + propertiesProposal.setEntityType(Constants.DATA_CONTRACT_ENTITY_NAME); + propertiesProposal.setSystemMetadata( + new SystemMetadata().setProperties(new StringMap(ImmutableMap.of("appSource", "ui")))); + propertiesProposal.setAspectName(Constants.DATA_CONTRACT_PROPERTIES_ASPECT_NAME); + propertiesProposal.setAspect(GenericRecordUtils.serializeAspect(props)); + propertiesProposal.setChangeType(ChangeType.UPSERT); + + final MetadataChangeProposal statusProposal = new MetadataChangeProposal(); + statusProposal.setEntityUrn(dataContractUrn); + statusProposal.setEntityType(Constants.DATA_CONTRACT_ENTITY_NAME); + statusProposal.setSystemMetadata( + new SystemMetadata().setProperties(new StringMap(ImmutableMap.of("appSource", "ui")))); + statusProposal.setAspectName(Constants.DATA_CONTRACT_STATUS_ASPECT_NAME); + statusProposal.setAspect(GenericRecordUtils.serializeAspect(status)); + statusProposal.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockClient, Mockito.times(1)) + .batchIngestProposals( + any(OperationContext.class), + Mockito.eq(ImmutableList.of(propertiesProposal, statusProposal)), + Mockito.eq(false)); + + Assert.assertEquals(result.getUrn(), TEST_CONTRACT_URN.toString()); + } + + @Test + public void testGetSuccessUpdate() throws Exception { + + DataContractProperties props = new DataContractProperties(); + props.setEntity(TEST_DATASET_URN); + props.setDataQuality( + new DataQualityContractArray( + ImmutableList.of(new DataQualityContract().setAssertion(TEST_QUALITY_ASSERTION_URN)))); + props.setFreshness( + new FreshnessContractArray( + ImmutableList.of(new FreshnessContract().setAssertion(TEST_FRESHNESS_ASSERTION_URN)))); + props.setSchema( + new SchemaContractArray( + ImmutableList.of(new SchemaContract().setAssertion(TEST_SCHEMA_ASSERTION_URN)))); + + DataContractStatus status = new DataContractStatus(); + status.setState(com.linkedin.datacontract.DataContractState.ACTIVE); + + // Update resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + GraphClient mockGraphClient = Mockito.mock(GraphClient.class); + initMockGraphClient(mockGraphClient, TEST_CONTRACT_URN); + initMockEntityClient(mockClient, TEST_CONTRACT_URN, props); // Contract Exists + UpsertDataContractResolver resolver = + new UpsertDataContractResolver(mockClient, mockGraphClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_VALID_UPDATE_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + DataContract result = resolver.get(mockEnv).get(); + + final MetadataChangeProposal propertiesProposal = new MetadataChangeProposal(); + propertiesProposal.setEntityUrn(TEST_CONTRACT_URN); + propertiesProposal.setEntityType(Constants.DATA_CONTRACT_ENTITY_NAME); + propertiesProposal.setSystemMetadata( + new SystemMetadata().setProperties(new StringMap(ImmutableMap.of("appSource", "ui")))); + propertiesProposal.setAspectName(Constants.DATA_CONTRACT_PROPERTIES_ASPECT_NAME); + propertiesProposal.setAspect(GenericRecordUtils.serializeAspect(props)); + propertiesProposal.setChangeType(ChangeType.UPSERT); + + final MetadataChangeProposal statusProposal = new MetadataChangeProposal(); + statusProposal.setEntityUrn(TEST_CONTRACT_URN); + statusProposal.setEntityType(Constants.DATA_CONTRACT_ENTITY_NAME); + statusProposal.setSystemMetadata( + new SystemMetadata().setProperties(new StringMap(ImmutableMap.of("appSource", "ui")))); + statusProposal.setAspectName(Constants.DATA_CONTRACT_STATUS_ASPECT_NAME); + statusProposal.setAspect(GenericRecordUtils.serializeAspect(status)); + statusProposal.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockClient, Mockito.times(1)) + .batchIngestProposals( + any(OperationContext.class), + Mockito.eq(ImmutableList.of(propertiesProposal, statusProposal)), + Mockito.eq(false)); + + Assert.assertEquals(result.getUrn(), TEST_CONTRACT_URN.toString()); + } + + @Test + public void testGetFailureEntityDoesNotExist() throws Exception { + // Update resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + GraphClient mockGraphClient = Mockito.mock(GraphClient.class); + initMockGraphClient(mockGraphClient, TEST_CONTRACT_URN); + Mockito.when(mockClient.exists(any(OperationContext.class), Mockito.eq(TEST_DATASET_URN))) + .thenReturn(false); + UpsertDataContractResolver resolver = + new UpsertDataContractResolver(mockClient, mockGraphClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_CREATE_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Assert.assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + @Test + public void testGetFailureAssertionDoesNotExist() throws Exception { + // Update resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + GraphClient mockGraphClient = Mockito.mock(GraphClient.class); + initMockGraphClient(mockGraphClient, TEST_CONTRACT_URN); + Mockito.when(mockClient.exists(any(OperationContext.class), Mockito.eq(TEST_DATASET_URN))) + .thenReturn(true); + Mockito.when( + mockClient.exists( + any(OperationContext.class), Mockito.eq(TEST_FRESHNESS_ASSERTION_URN))) + .thenReturn(false); + Mockito.when( + mockClient.exists(any(OperationContext.class), Mockito.eq(TEST_QUALITY_ASSERTION_URN))) + .thenReturn(false); + Mockito.when( + mockClient.exists(any(OperationContext.class), Mockito.eq(TEST_SCHEMA_ASSERTION_URN))) + .thenReturn(false); + UpsertDataContractResolver resolver = + new UpsertDataContractResolver(mockClient, mockGraphClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_CREATE_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Assert.assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + @Test + public void testGetUnauthorized() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + GraphClient mockGraphClient = Mockito.mock(GraphClient.class); + UpsertDataContractResolver resolver = + new UpsertDataContractResolver(mockClient, mockGraphClient); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_CREATE_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockClient, Mockito.times(0)) + .ingestProposal(any(OperationContext.class), Mockito.any()); + } + + @Test + public void testGetEntityClientException() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + GraphClient mockGraphClient = Mockito.mock(GraphClient.class); + Mockito.doThrow(RemoteInvocationException.class) + .when(mockClient) + .ingestProposal(any(OperationContext.class), Mockito.any(), Mockito.eq(false)); + UpsertDataContractResolver resolver = + new UpsertDataContractResolver(mockClient, mockGraphClient); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_CREATE_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + private void initMockGraphClient(GraphClient client, Urn existingContractUrn) { + if (existingContractUrn != null) { + Mockito.when( + client.getRelatedEntities( + Mockito.eq(TEST_DATASET_URN.toString()), + Mockito.eq(ImmutableList.of(CONTRACT_FOR_RELATIONSHIP)), + Mockito.eq(RelationshipDirection.INCOMING), + Mockito.eq(0), + Mockito.eq(1), + Mockito.anyString())) + .thenReturn( + new EntityRelationships() + .setTotal(1) + .setCount(1) + .setStart(0) + .setRelationships( + new EntityRelationshipArray( + ImmutableList.of( + new EntityRelationship() + .setEntity(existingContractUrn) + .setType(CONTRACT_FOR_RELATIONSHIP) + .setCreated( + new AuditStamp().setActor(TEST_ACTOR_URN).setTime(0L)))))); + } else { + Mockito.when( + client.getRelatedEntities( + Mockito.eq(TEST_DATASET_URN.toString()), + Mockito.eq(ImmutableList.of(CONTRACT_FOR_RELATIONSHIP)), + Mockito.eq(RelationshipDirection.INCOMING), + Mockito.eq(0), + Mockito.eq(1), + Mockito.anyString())) + .thenReturn( + new EntityRelationships() + .setTotal(0) + .setCount(0) + .setStart(0) + .setRelationships(new EntityRelationshipArray(Collections.emptyList()))); + } + } + + private void initMockEntityClient( + EntityClient client, Urn existingContractUrn, DataContractProperties newContractProperties) + throws Exception { + if (existingContractUrn != null) { + Mockito.when(client.exists(any(OperationContext.class), Mockito.eq(existingContractUrn))) + .thenReturn(true); + } + Mockito.when(client.exists(any(OperationContext.class), Mockito.eq(TEST_DATASET_URN))) + .thenReturn(true); + Mockito.when(client.exists(any(OperationContext.class), Mockito.eq(TEST_QUALITY_ASSERTION_URN))) + .thenReturn(true); + Mockito.when( + client.exists(any(OperationContext.class), Mockito.eq(TEST_FRESHNESS_ASSERTION_URN))) + .thenReturn(true); + Mockito.when(client.exists(any(OperationContext.class), Mockito.eq(TEST_SCHEMA_ASSERTION_URN))) + .thenReturn(true); + + Mockito.when( + client.getV2( + any(OperationContext.class), + Mockito.eq(Constants.DATA_CONTRACT_ENTITY_NAME), + Mockito.eq(TEST_CONTRACT_URN), + Mockito.eq(null))) + .thenReturn( + new EntityResponse() + .setUrn(TEST_CONTRACT_URN) + .setAspects( + new EnvelopedAspectMap( + ImmutableMap.of( + Constants.DATA_CONTRACT_PROPERTIES_ASPECT_NAME, + new EnvelopedAspect() + .setType(AspectType.VERSIONED) + .setName(Constants.DATA_CONTRACT_ENTITY_NAME) + .setValue(new Aspect(newContractProperties.data())))))); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/CreateFormResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/CreateFormResolverTest.java new file mode 100644 index 0000000000000..65f51830ee148 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/CreateFormResolverTest.java @@ -0,0 +1,116 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.*; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.CreateFormInput; +import com.linkedin.datahub.graphql.generated.Form; +import com.linkedin.datahub.graphql.generated.FormType; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.form.FormInfo; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.service.FormService; +import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class CreateFormResolverTest { + private static final String TEST_FORM_URN = "urn:li:form:1"; + + private static final CreateFormInput TEST_INPUT = + new CreateFormInput(null, "test name", null, FormType.VERIFICATION, new ArrayList<>(), null); + + @Test + public void testGetSuccess() throws Exception { + FormService mockFormService = initMockFormService(true); + EntityClient mockEntityClient = initMockEntityClient(); + CreateFormResolver resolver = new CreateFormResolver(mockEntityClient, mockFormService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Form form = resolver.get(mockEnv).get(); + + assertEquals(form.getUrn(), TEST_FORM_URN); + + // Validate that we called create on the service + Mockito.verify(mockFormService, Mockito.times(1)) + .createForm(any(), any(FormInfo.class), Mockito.eq(null)); + } + + @Test + public void testGetUnauthorized() throws Exception { + FormService mockFormService = initMockFormService(true); + EntityClient mockEntityClient = initMockEntityClient(); + CreateFormResolver resolver = new CreateFormResolver(mockEntityClient, mockFormService); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call create on the service + Mockito.verify(mockFormService, Mockito.times(0)) + .createForm(any(), any(FormInfo.class), Mockito.eq(null)); + } + + @Test + public void testGetFailure() throws Exception { + FormService mockFormService = initMockFormService(false); + EntityClient mockEntityClient = initMockEntityClient(); + CreateFormResolver resolver = new CreateFormResolver(mockEntityClient, mockFormService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we called create on the service + Mockito.verify(mockFormService, Mockito.times(1)) + .createForm(any(), any(FormInfo.class), Mockito.eq(null)); + } + + private FormService initMockFormService(final boolean shouldSucceed) throws Exception { + FormService service = Mockito.mock(FormService.class); + + if (shouldSucceed) { + Mockito.when(service.createForm(any(), Mockito.any(), Mockito.any())) + .thenReturn(UrnUtils.getUrn("urn:li:form:1")); + } else { + Mockito.when(service.createForm(any(), Mockito.any(), Mockito.any())) + .thenThrow(new RuntimeException()); + } + + return service; + } + + private EntityClient initMockEntityClient() throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + EntityResponse response = new EntityResponse(); + response.setEntityName(Constants.FORM_ENTITY_NAME); + response.setUrn(UrnUtils.getUrn(TEST_FORM_URN)); + response.setAspects(new EnvelopedAspectMap()); + Mockito.when( + client.getV2(any(), Mockito.eq(Constants.FORM_ENTITY_NAME), any(), Mockito.eq(null))) + .thenReturn(response); + + return client; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/DeleteFormResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/DeleteFormResolverTest.java new file mode 100644 index 0000000000000..ded79ed9a0018 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/DeleteFormResolverTest.java @@ -0,0 +1,90 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DeleteFormInput; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class DeleteFormResolverTest { + private static final String TEST_FORM_URN = "urn:li:form:1"; + + private static final DeleteFormInput TEST_INPUT = new DeleteFormInput(TEST_FORM_URN); + + @Test + public void testGetSuccess() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + DeleteFormResolver resolver = new DeleteFormResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Boolean success = resolver.get(mockEnv).get(); + assertTrue(success); + + // Validate that we called delete + Mockito.verify(mockEntityClient, Mockito.times(1)) + .deleteEntity(any(), Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN))); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + DeleteFormResolver resolver = new DeleteFormResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call delete and delete references + Mockito.verify(mockEntityClient, Mockito.times(0)) + .deleteEntity(any(), Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN))); + Mockito.verify(mockEntityClient, Mockito.times(0)) + .deleteEntityReferences(any(), Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN))); + } + + @Test + public void testGetFailure() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(false); + DeleteFormResolver resolver = new DeleteFormResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that deleteEntity was called, but since it failed, delete references was not called + Mockito.verify(mockEntityClient, Mockito.times(1)) + .deleteEntity(any(), Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN))); + Mockito.verify(mockEntityClient, Mockito.times(0)) + .deleteEntityReferences(any(), Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN))); + } + + private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + if (!shouldSucceed) { + Mockito.doThrow(new RemoteInvocationException()).when(client).deleteEntity(any(), any()); + } + return client; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/UpdateFormResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/UpdateFormResolverTest.java new file mode 100644 index 0000000000000..6a4b99742f7fd --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/UpdateFormResolverTest.java @@ -0,0 +1,105 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.*; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.Form; +import com.linkedin.datahub.graphql.generated.UpdateFormInput; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class UpdateFormResolverTest { + private static final String TEST_FORM_URN = "urn:li:form:1"; + + private static final UpdateFormInput TEST_INPUT = + new UpdateFormInput(TEST_FORM_URN, "new name", null, null, null, null, null); + + @Test + public void testGetSuccess() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + UpdateFormResolver resolver = new UpdateFormResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Form form = resolver.get(mockEnv).get(); + + assertEquals(form.getUrn(), TEST_FORM_URN); + + // Validate that we called ingest + Mockito.verify(mockEntityClient, Mockito.times(1)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + UpdateFormResolver resolver = new UpdateFormResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call ingest + Mockito.verify(mockEntityClient, Mockito.times(0)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetFailure() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(false); + UpdateFormResolver resolver = new UpdateFormResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that ingest was called, but that caused a failure + Mockito.verify(mockEntityClient, Mockito.times(1)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + EntityResponse response = new EntityResponse(); + response.setEntityName(Constants.FORM_ENTITY_NAME); + response.setUrn(UrnUtils.getUrn(TEST_FORM_URN)); + response.setAspects(new EnvelopedAspectMap()); + if (shouldSucceed) { + Mockito.when( + client.getV2(any(), Mockito.eq(Constants.FORM_ENTITY_NAME), any(), Mockito.eq(null))) + .thenReturn(response); + } else { + Mockito.when( + client.getV2(any(), Mockito.eq(Constants.FORM_ENTITY_NAME), any(), Mockito.eq(null))) + .thenThrow(new RemoteInvocationException()); + } + + Mockito.when(client.exists(any(), Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN)))).thenReturn(true); + + return client; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolverTest.java index 9ed1d5001b75c..70b427a1606f1 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolverTest.java @@ -169,6 +169,6 @@ private Filter buildFilter(@Nullable QuerySource source, @Nullable String entity FilterOperator.EQUAL)); } criteria.setAnd(andConditions); - return ResolverUtils.buildFilter(Collections.emptyList(), ImmutableList.of(criteria)); + return ResolverUtils.buildFilter(Collections.emptyList(), ImmutableList.of(criteria), null); } } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java new file mode 100644 index 0000000000000..72cdb78542e41 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java @@ -0,0 +1,126 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.CreateStructuredPropertyInput; +import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class CreateStructuredPropertyResolverTest { + private static final String TEST_STRUCTURED_PROPERTY_URN = "urn:li:structuredProperty:1"; + + private static final CreateStructuredPropertyInput TEST_INPUT = + new CreateStructuredPropertyInput( + null, + "io.acryl.test", + "Display Name", + "description", + true, + null, + null, + null, + null, + new ArrayList<>()); + + @Test + public void testGetSuccess() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + CreateStructuredPropertyResolver resolver = + new CreateStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + StructuredPropertyEntity prop = resolver.get(mockEnv).get(); + + assertEquals(prop.getUrn(), TEST_STRUCTURED_PROPERTY_URN); + + // Validate that we called ingest + Mockito.verify(mockEntityClient, Mockito.times(1)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + CreateStructuredPropertyResolver resolver = + new CreateStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call ingest + Mockito.verify(mockEntityClient, Mockito.times(0)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetFailure() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(false); + CreateStructuredPropertyResolver resolver = + new CreateStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that ingest was called, but that caused a failure + Mockito.verify(mockEntityClient, Mockito.times(1)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + EntityResponse response = new EntityResponse(); + response.setEntityName(Constants.STRUCTURED_PROPERTY_ENTITY_NAME); + response.setUrn(UrnUtils.getUrn(TEST_STRUCTURED_PROPERTY_URN)); + response.setAspects(new EnvelopedAspectMap()); + if (shouldSucceed) { + Mockito.when( + client.getV2( + any(), + Mockito.eq(Constants.STRUCTURED_PROPERTY_ENTITY_NAME), + any(), + Mockito.eq(null))) + .thenReturn(response); + } else { + Mockito.when( + client.getV2( + any(), + Mockito.eq(Constants.STRUCTURED_PROPERTY_ENTITY_NAME), + any(), + Mockito.eq(null))) + .thenThrow(new RemoteInvocationException()); + } + + return client; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolverTest.java new file mode 100644 index 0000000000000..f7882bb2c93a8 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolverTest.java @@ -0,0 +1,123 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.assertThrows; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.RemoveStructuredPropertiesInput; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.structured.StructuredProperties; +import com.linkedin.structured.StructuredPropertyValueAssignmentArray; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class RemoveStructuredPropertiesResolverTest { + + private static final String TEST_DATASET_URN = + "urn:li:dataset:(urn:li:dataPlatform:hive,name,PROD)"; + private static final String PROPERTY_URN_1 = "urn:li:structuredProperty:test1"; + private static final String PROPERTY_URN_2 = "urn:li:structuredProperty:test2"; + + private static final RemoveStructuredPropertiesInput TEST_INPUT = + new RemoveStructuredPropertiesInput( + TEST_DATASET_URN, ImmutableList.of(PROPERTY_URN_1, PROPERTY_URN_2)); + + @Test + public void testGetSuccess() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(); + RemoveStructuredPropertiesResolver resolver = + new RemoveStructuredPropertiesResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + resolver.get(mockEnv).get(); + + // Validate that we called ingest + Mockito.verify(mockEntityClient, Mockito.times(1)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(); + RemoveStructuredPropertiesResolver resolver = + new RemoveStructuredPropertiesResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call ingest + Mockito.verify(mockEntityClient, Mockito.times(0)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetThrowsError() throws Exception { + // if the entity you are trying to remove properties from doesn't exist + EntityClient mockEntityClient = Mockito.mock(EntityClient.class); + Mockito.when(mockEntityClient.exists(any(), Mockito.eq(UrnUtils.getUrn(TEST_DATASET_URN)))) + .thenReturn(false); + RemoveStructuredPropertiesResolver resolver = + new RemoveStructuredPropertiesResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call ingest + Mockito.verify(mockEntityClient, Mockito.times(0)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + private EntityClient initMockEntityClient() throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + EntityResponse response = new EntityResponse(); + response.setEntityName(Constants.DATASET_ENTITY_NAME); + response.setUrn(UrnUtils.getUrn(TEST_DATASET_URN)); + final EnvelopedAspectMap aspectMap = new EnvelopedAspectMap(); + StructuredProperties properties = new StructuredProperties(); + properties.setProperties(new StructuredPropertyValueAssignmentArray()); + aspectMap.put( + STRUCTURED_PROPERTIES_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(properties.data()))); + response.setAspects(aspectMap); + Mockito.when( + client.getV2( + any(), + Mockito.eq(Constants.DATASET_ENTITY_NAME), + Mockito.eq(UrnUtils.getUrn(TEST_DATASET_URN)), + Mockito.eq(ImmutableSet.of(Constants.STRUCTURED_PROPERTIES_ASPECT_NAME)))) + .thenReturn(response); + Mockito.when(client.exists(any(), Mockito.eq(UrnUtils.getUrn(TEST_DATASET_URN)))) + .thenReturn(true); + + return client; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java new file mode 100644 index 0000000000000..971a53de9473b --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java @@ -0,0 +1,123 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.generated.UpdateStructuredPropertyInput; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class UpdateStructuredPropertyResolverTest { + private static final String TEST_STRUCTURED_PROPERTY_URN = "urn:li:structuredProperty:1"; + + private static final UpdateStructuredPropertyInput TEST_INPUT = + new UpdateStructuredPropertyInput( + TEST_STRUCTURED_PROPERTY_URN, + "New Display Name", + "new description", + true, + null, + null, + null, + null); + + @Test + public void testGetSuccess() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + UpdateStructuredPropertyResolver resolver = + new UpdateStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + StructuredPropertyEntity prop = resolver.get(mockEnv).get(); + + assertEquals(prop.getUrn(), TEST_STRUCTURED_PROPERTY_URN); + + // Validate that we called ingest + Mockito.verify(mockEntityClient, Mockito.times(1)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + UpdateStructuredPropertyResolver resolver = + new UpdateStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call ingest + Mockito.verify(mockEntityClient, Mockito.times(0)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetFailure() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(false); + UpdateStructuredPropertyResolver resolver = + new UpdateStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that ingest was called, but that caused a failure + Mockito.verify(mockEntityClient, Mockito.times(1)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + EntityResponse response = new EntityResponse(); + response.setEntityName(Constants.STRUCTURED_PROPERTY_ENTITY_NAME); + response.setUrn(UrnUtils.getUrn(TEST_STRUCTURED_PROPERTY_URN)); + response.setAspects(new EnvelopedAspectMap()); + if (shouldSucceed) { + Mockito.when( + client.getV2( + any(), + Mockito.eq(Constants.STRUCTURED_PROPERTY_ENTITY_NAME), + any(), + Mockito.eq(null))) + .thenReturn(response); + } else { + Mockito.when( + client.getV2( + any(), + Mockito.eq(Constants.STRUCTURED_PROPERTY_ENTITY_NAME), + any(), + Mockito.eq(null))) + .thenThrow(new RemoteInvocationException()); + } + + return client; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java index 443050456f3fd..701ddd84c173e 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java @@ -2,6 +2,7 @@ import static com.linkedin.datahub.graphql.TestUtils.*; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import static org.testng.Assert.*; import com.google.common.collect.ImmutableList; @@ -17,6 +18,7 @@ import com.linkedin.datahub.graphql.generated.FilterOperator; import com.linkedin.datahub.graphql.generated.LogicalOperator; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; @@ -169,7 +171,7 @@ public void testMapDefinition() throws Exception { // the keyword mapping. .setCondition(Condition.CONTAIN)))))))); - assertEquals(ViewUtils.mapDefinition(input), expectedResult); + assertEquals(ViewUtils.mapDefinition(input, mock(AspectRetriever.class)), expectedResult); } private static ViewService initViewService(DataHubViewType viewType) { diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/AssertionMapperTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/AssertionMapperTest.java new file mode 100644 index 0000000000000..82f4fe687bf76 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/AssertionMapperTest.java @@ -0,0 +1,415 @@ +package com.linkedin.datahub.graphql.types.assertion; + +import static org.testng.Assert.assertEquals; + +import com.google.common.collect.ImmutableList; +import com.linkedin.assertion.AssertionInfo; +import com.linkedin.assertion.AssertionSource; +import com.linkedin.assertion.AssertionSourceType; +import com.linkedin.assertion.AssertionStdAggregation; +import com.linkedin.assertion.AssertionStdOperator; +import com.linkedin.assertion.AssertionStdParameter; +import com.linkedin.assertion.AssertionStdParameterType; +import com.linkedin.assertion.AssertionStdParameters; +import com.linkedin.assertion.AssertionType; +import com.linkedin.assertion.CustomAssertionInfo; +import com.linkedin.assertion.DatasetAssertionInfo; +import com.linkedin.assertion.DatasetAssertionScope; +import com.linkedin.assertion.FreshnessAssertionInfo; +import com.linkedin.assertion.FreshnessAssertionSchedule; +import com.linkedin.assertion.FreshnessAssertionScheduleType; +import com.linkedin.assertion.FreshnessAssertionType; +import com.linkedin.assertion.FreshnessCronSchedule; +import com.linkedin.assertion.SchemaAssertionCompatibility; +import com.linkedin.assertion.SchemaAssertionInfo; +import com.linkedin.common.GlobalTags; +import com.linkedin.common.TagAssociationArray; +import com.linkedin.common.UrnArray; +import com.linkedin.common.url.Url; +import com.linkedin.common.urn.TagUrn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.DataMap; +import com.linkedin.data.template.StringMap; +import com.linkedin.datahub.graphql.generated.Assertion; +import com.linkedin.datahub.graphql.generated.FixedIntervalSchedule; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.metadata.Constants; +import com.linkedin.schema.MySqlDDL; +import com.linkedin.schema.SchemaField; +import com.linkedin.schema.SchemaFieldArray; +import com.linkedin.schema.SchemaFieldDataType; +import com.linkedin.schema.SchemaMetadata; +import com.linkedin.schema.StringType; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class AssertionMapperTest { + + @Test + public void testMapDatasetAssertion() { + // Case 1: Without nullable fields + AssertionInfo input = createFreshnessAssertionInfoWithoutNullableFields(); + EntityResponse datasetAssertionEntityResponse = createAssertionInfoEntityResponse(input); + Assertion output = AssertionMapper.map(null, datasetAssertionEntityResponse); + verifyAssertionInfo(input, output); + + // Case 2: With nullable fields + input = createFreshnessAssertionInfoWithNullableFields(); + EntityResponse datasetAssertionEntityResponseWithNullables = + createAssertionInfoEntityResponse(input); + output = AssertionMapper.map(null, datasetAssertionEntityResponseWithNullables); + verifyAssertionInfo(input, output); + } + + @Test + public void testMapTags() throws Exception { + HashMap aspects = new HashMap<>(); + AssertionInfo info = createFreshnessAssertionInfoWithoutNullableFields(); + + EnvelopedAspect envelopedTagsAspect = new EnvelopedAspect(); + GlobalTags tags = new GlobalTags(); + tags.setTags( + new TagAssociationArray( + new TagAssociationArray( + Collections.singletonList( + new com.linkedin.common.TagAssociation() + .setTag(TagUrn.createFromString("urn:li:tag:test")))))); + envelopedTagsAspect.setValue(new Aspect(tags.data())); + + aspects.put(Constants.ASSERTION_INFO_ASPECT_NAME, createEnvelopedAspect(info.data())); + aspects.put(Constants.GLOBAL_TAGS_ASPECT_NAME, createEnvelopedAspect(tags.data())); + EntityResponse response = createEntityResponse(aspects); + + Assertion assertion = AssertionMapper.map(null, response); + assertEquals(assertion.getTags().getTags().size(), 1); + assertEquals( + assertion.getTags().getTags().get(0).getTag().getUrn().toString(), "urn:li:tag:test"); + } + + @Test + public void testMapFreshnessAssertion() { + // Case 1: Without nullable fields + AssertionInfo inputInfo = createFreshnessAssertionInfoWithoutNullableFields(); + + EntityResponse freshnessAssertionEntityResponse = createAssertionInfoEntityResponse(inputInfo); + Assertion output = AssertionMapper.map(null, freshnessAssertionEntityResponse); + verifyAssertionInfo(inputInfo, output); + + // Case 2: With nullable fields + inputInfo = createDatasetAssertionInfoWithNullableFields(); + EntityResponse freshnessAssertionEntityResponseWithNullables = + createAssertionInfoEntityResponse(inputInfo); + output = AssertionMapper.map(null, freshnessAssertionEntityResponseWithNullables); + verifyAssertionInfo(inputInfo, output); + } + + @Test + public void testMapDataSchemaAssertion() { + AssertionInfo input = createSchemaAssertion(); + EntityResponse schemaAssertionEntityResponse = createAssertionInfoEntityResponse(input); + Assertion output = AssertionMapper.map(null, schemaAssertionEntityResponse); + verifyAssertionInfo(input, output); + } + + @Test + public void testMapCustomAssertion() { + // Case 1: Without nullable fields + AssertionInfo input = createCustomAssertionInfoWithoutNullableFields(); + EntityResponse customAssertionEntityResponse = createAssertionInfoEntityResponse(input); + Assertion output = AssertionMapper.map(null, customAssertionEntityResponse); + verifyAssertionInfo(input, output); + + // Case 2: With nullable fields + input = createCustomAssertionInfoWithNullableFields(); + EntityResponse customAssertionEntityResponseWithNullables = + createAssertionInfoEntityResponse(input); + output = AssertionMapper.map(null, customAssertionEntityResponseWithNullables); + verifyAssertionInfo(input, output); + } + + private void verifyAssertionInfo(AssertionInfo input, Assertion output) { + Assert.assertNotNull(output); + Assert.assertNotNull(output.getInfo()); + Assert.assertEquals( + output.getInfo().getType().toString(), output.getInfo().getType().toString()); + + if (input.hasDatasetAssertion()) { + verifyDatasetAssertion(input.getDatasetAssertion(), output.getInfo().getDatasetAssertion()); + } + + if (input.hasExternalUrl()) { + Assert.assertEquals(input.getExternalUrl().toString(), output.getInfo().getExternalUrl()); + } + + if (input.hasFreshnessAssertion()) { + verifyFreshnessAssertion( + input.getFreshnessAssertion(), output.getInfo().getFreshnessAssertion()); + } + + if (input.hasSchemaAssertion()) { + verifySchemaAssertion(input.getSchemaAssertion(), output.getInfo().getSchemaAssertion()); + } + + if (input.hasSource()) { + verifySource(input.getSource(), output.getInfo().getSource()); + } + + if (input.hasCustomAssertion()) { + verifyCustomAssertion(input.getCustomAssertion(), output.getInfo().getCustomAssertion()); + } + } + + private void verifyDatasetAssertion( + DatasetAssertionInfo input, + com.linkedin.datahub.graphql.generated.DatasetAssertionInfo output) { + Assert.assertEquals(output.getOperator().toString(), input.getOperator().toString()); + Assert.assertEquals(output.getOperator().toString(), input.getOperator().toString()); + Assert.assertEquals(output.getScope().toString(), input.getScope().toString()); + Assert.assertEquals(output.getDatasetUrn(), input.getDataset().toString()); + if (input.hasAggregation()) { + Assert.assertEquals(output.getAggregation().toString(), input.getAggregation().toString()); + } + if (input.hasNativeType()) { + Assert.assertEquals(output.getNativeType(), input.getNativeType().toString()); + } + if (input.hasLogic()) { + Assert.assertEquals(output.getLogic(), input.getLogic()); + } + if (input.hasFields()) { + Assert.assertTrue( + input.getFields().stream() + .allMatch( + field -> + output.getFields().stream() + .anyMatch(outField -> field.toString().equals(outField.getUrn())))); + } + } + + private void verifyFreshnessAssertion( + FreshnessAssertionInfo input, + com.linkedin.datahub.graphql.generated.FreshnessAssertionInfo output) { + Assert.assertEquals(output.getType().toString(), input.getType().toString()); + Assert.assertEquals(output.getEntityUrn(), input.getEntity().toString()); + if (input.hasSchedule()) { + verifyFreshnessSchedule(input.getSchedule(), output.getSchedule()); + } + } + + private void verifySchemaAssertion( + SchemaAssertionInfo input, + com.linkedin.datahub.graphql.generated.SchemaAssertionInfo output) { + Assert.assertEquals(output.getEntityUrn(), input.getEntity().toString()); + Assert.assertEquals(output.getCompatibility().toString(), input.getCompatibility().toString()); + Assert.assertEquals( + output.getSchema().getFields().size(), input.getSchema().getFields().size()); + } + + private void verifyCustomAssertion( + CustomAssertionInfo input, + com.linkedin.datahub.graphql.generated.CustomAssertionInfo output) { + Assert.assertEquals(output.getEntityUrn(), input.getEntity().toString()); + Assert.assertEquals(output.getType(), input.getType()); + if (input.hasLogic()) { + Assert.assertEquals(output.getLogic(), input.getLogic()); + } + if (input.hasField()) { + Assert.assertEquals(output.getField().getPath(), input.getField().getEntityKey().get(1)); + } + } + + private void verifyCronSchedule( + FreshnessCronSchedule input, + com.linkedin.datahub.graphql.generated.FreshnessCronSchedule output) { + Assert.assertEquals(output.getCron(), input.getCron()); + Assert.assertEquals(output.getTimezone(), input.getTimezone()); + if (input.hasWindowStartOffsetMs()) { + Assert.assertEquals(output.getWindowStartOffsetMs(), input.getWindowStartOffsetMs()); + } + } + + private void verifyFreshnessSchedule( + FreshnessAssertionSchedule input, + com.linkedin.datahub.graphql.generated.FreshnessAssertionSchedule output) { + Assert.assertEquals(output.getType().toString(), input.getType().toString()); + if (input.hasCron()) { + verifyCronSchedule(input.getCron(), output.getCron()); + } + if (input.hasFixedInterval()) { + verifyFixedIntervalSchedule(input.getFixedInterval(), output.getFixedInterval()); + } + } + + private void verifyFixedIntervalSchedule( + com.linkedin.assertion.FixedIntervalSchedule input, FixedIntervalSchedule output) { + Assert.assertEquals(output.getMultiple(), (int) input.getMultiple()); + Assert.assertEquals(output.getUnit().toString(), input.getUnit().toString()); + } + + private void verifySource( + AssertionSource input, com.linkedin.datahub.graphql.generated.AssertionSource output) { + Assert.assertEquals(output.getType().toString(), input.getType().toString()); + } + + private EntityResponse createAssertionInfoEntityResponse(final AssertionInfo info) { + HashMap aspects = new HashMap<>(); + aspects.put(Constants.ASSERTION_INFO_ASPECT_NAME, createEnvelopedAspect(info.data())); + + return createEntityResponse(aspects); + } + + private EntityResponse createEntityResponse(Map aspects) { + EntityResponse entityResponse = new EntityResponse(); + entityResponse.setUrn(UrnUtils.getUrn("urn:li:assertion:1")); + entityResponse.setAspects(new EnvelopedAspectMap(new HashMap<>())); + aspects.forEach( + (aspectName, envelopedAspect) -> { + entityResponse.getAspects().put(aspectName, envelopedAspect); + }); + + return entityResponse; + } + + private EnvelopedAspect createEnvelopedAspect(DataMap dataMap) { + EnvelopedAspect envelopedAspect = new EnvelopedAspect(); + envelopedAspect.setValue(new Aspect(dataMap)); + return envelopedAspect; + } + + private AssertionInfo createDatasetAssertionInfoWithoutNullableFields() { + AssertionInfo info = new AssertionInfo(); + info.setType(com.linkedin.assertion.AssertionType.DATASET); + DatasetAssertionInfo datasetAssertionInfo = new DatasetAssertionInfo(); + datasetAssertionInfo.setDataset(UrnUtils.getUrn("urn:li:dataset:1")); + datasetAssertionInfo.setScope(DatasetAssertionScope.DATASET_COLUMN); + datasetAssertionInfo.setOperator(AssertionStdOperator.GREATER_THAN); + info.setDatasetAssertion(datasetAssertionInfo); + return info; + } + + private AssertionInfo createDatasetAssertionInfoWithNullableFields() { + AssertionInfo infoWithoutNullables = createDatasetAssertionInfoWithoutNullableFields(); + DatasetAssertionInfo baseInfo = infoWithoutNullables.getDatasetAssertion(); + baseInfo.setFields( + new UrnArray( + Arrays.asList( + UrnUtils.getUrn( + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,name,PROD),field)")))); + baseInfo.setAggregation(AssertionStdAggregation.SUM); + baseInfo.setParameters(createAssertionStdParameters()); + baseInfo.setNativeType("native_type"); + baseInfo.setNativeParameters(new StringMap(Collections.singletonMap("key", "value"))); + baseInfo.setLogic("sample_logic"); + infoWithoutNullables.setSource( + new AssertionSource().setType(com.linkedin.assertion.AssertionSourceType.INFERRED)); + return infoWithoutNullables; + } + + private AssertionInfo createFreshnessAssertionInfoWithoutNullableFields() { + AssertionInfo info = new AssertionInfo(); + info.setType(AssertionType.FRESHNESS); + FreshnessAssertionInfo freshnessAssertionInfo = new FreshnessAssertionInfo(); + freshnessAssertionInfo.setEntity( + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,name,PROD)")); + freshnessAssertionInfo.setType(FreshnessAssertionType.DATASET_CHANGE); + info.setFreshnessAssertion(freshnessAssertionInfo); + return info; + } + + private AssertionInfo createFreshnessAssertionInfoWithNullableFields() { + AssertionInfo infoWithoutNullables = createFreshnessAssertionInfoWithoutNullableFields(); + FreshnessAssertionInfo baseInfo = infoWithoutNullables.getFreshnessAssertion(); + baseInfo.setSchedule(createFreshnessAssertionSchedule()); + infoWithoutNullables.setSource( + new AssertionSource().setType(com.linkedin.assertion.AssertionSourceType.INFERRED)); + return infoWithoutNullables; + } + + private AssertionInfo createSchemaAssertion() { + AssertionInfo info = new AssertionInfo(); + info.setType(AssertionType.DATA_SCHEMA); + SchemaAssertionInfo schemaAssertionInfo = new SchemaAssertionInfo(); + schemaAssertionInfo.setEntity(UrnUtils.getUrn("urn:li:dataset:1")); + schemaAssertionInfo.setCompatibility(SchemaAssertionCompatibility.SUPERSET); + schemaAssertionInfo.setSchema( + new SchemaMetadata() + .setCluster("Test") + .setHash("Test") + .setPlatformSchema(SchemaMetadata.PlatformSchema.create(new MySqlDDL())) + .setFields( + new SchemaFieldArray( + ImmutableList.of( + new SchemaField() + .setType( + new SchemaFieldDataType() + .setType(SchemaFieldDataType.Type.create(new StringType()))) + .setNullable(false) + .setNativeDataType("string") + .setFieldPath("test"))))); + return info; + } + + private AssertionInfo createCustomAssertionInfoWithoutNullableFields() { + AssertionInfo info = new AssertionInfo(); + info.setType(AssertionType.CUSTOM); + CustomAssertionInfo customAssertionInfo = new CustomAssertionInfo(); + customAssertionInfo.setType("Custom Type 1"); + customAssertionInfo.setEntity(UrnUtils.getUrn("urn:li:dataset:1")); + info.setCustomAssertion(customAssertionInfo); + return info; + } + + private AssertionInfo createCustomAssertionInfoWithNullableFields() { + AssertionInfo info = new AssertionInfo(); + info.setType(AssertionType.CUSTOM); + info.setExternalUrl(new Url("https://xyz.com")); + info.setDescription("Description of custom assertion"); + CustomAssertionInfo customAssertionInfo = new CustomAssertionInfo(); + customAssertionInfo.setType("Custom Type 1"); + customAssertionInfo.setEntity( + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,name,PROD)")); + customAssertionInfo.setField( + UrnUtils.getUrn( + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,name,PROD),field)")); + customAssertionInfo.setLogic("custom logic"); + info.setCustomAssertion(customAssertionInfo); + info.setSource(new AssertionSource().setType(AssertionSourceType.EXTERNAL)); + + return info; + } + + private AssertionStdParameters createAssertionStdParameters() { + AssertionStdParameters parameters = new AssertionStdParameters(); + parameters.setValue(createAssertionStdParameter()); + parameters.setMinValue(createAssertionStdParameter()); + parameters.setMaxValue(createAssertionStdParameter()); + return parameters; + } + + private AssertionStdParameter createAssertionStdParameter() { + AssertionStdParameter parameter = new AssertionStdParameter(); + parameter.setType(AssertionStdParameterType.NUMBER); + parameter.setValue("100"); + return parameter; + } + + private FreshnessAssertionSchedule createFreshnessAssertionSchedule() { + FreshnessAssertionSchedule schedule = new FreshnessAssertionSchedule(); + schedule.setType(FreshnessAssertionScheduleType.CRON); + schedule.setCron(createCronSchedule()); + return schedule; + } + + private FreshnessCronSchedule createCronSchedule() { + FreshnessCronSchedule cronSchedule = new FreshnessCronSchedule(); + cronSchedule.setCron("0 0 * * *"); + cronSchedule.setTimezone("UTC"); + return cronSchedule; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/AssertionTypeTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/AssertionTypeTest.java index dd2b676a94130..33774690b7c7a 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/AssertionTypeTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/AssertionTypeTest.java @@ -7,6 +7,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.linkedin.assertion.AssertionAction; +import com.linkedin.assertion.AssertionActionArray; +import com.linkedin.assertion.AssertionActionType; +import com.linkedin.assertion.AssertionActions; import com.linkedin.assertion.AssertionInfo; import com.linkedin.assertion.AssertionType; import com.linkedin.common.DataPlatformInstance; @@ -48,6 +52,17 @@ public class AssertionTypeTest { new DataPlatformInstance() .setPlatform(new DataPlatformUrn("snowflake")) .setInstance(null, SetMode.IGNORE_NULL); + // Acryl SaaS Only + private static final AssertionActions TEST_ASSERTION_ACTIONS = + new AssertionActions() + .setOnSuccess( + new AssertionActionArray( + ImmutableList.of( + new AssertionAction().setType(AssertionActionType.RAISE_INCIDENT)))) + .setOnFailure( + new AssertionActionArray( + ImmutableList.of( + new AssertionAction().setType(AssertionActionType.RESOLVE_INCIDENT)))); private static final String TEST_ASSERTION_URN_2 = "urn:li:assertion:guid-2"; @@ -69,6 +84,9 @@ public void testBatchLoad() throws Exception { assertion1Aspects.put( Constants.ASSERTION_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(TEST_ASSERTION_INFO.data()))); + assertion1Aspects.put( + Constants.ASSERTION_ACTIONS_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(TEST_ASSERTION_ACTIONS.data()))); Mockito.when( client.batchGetV2( any(), @@ -112,6 +130,12 @@ public void testBatchLoad() throws Exception { assertEquals(assertion.getInfo().getType().toString(), AssertionType.DATASET.toString()); assertEquals(assertion.getInfo().getDatasetAssertion(), null); assertEquals(assertion.getPlatform().getUrn(), "urn:li:dataPlatform:snowflake"); + assertEquals( + assertion.getActions().getOnSuccess().get(0).getType(), + com.linkedin.datahub.graphql.generated.AssertionActionType.RAISE_INCIDENT); + assertEquals( + assertion.getActions().getOnFailure().get(0).getType(), + com.linkedin.datahub.graphql.generated.AssertionActionType.RESOLVE_INCIDENT); // Assert second element is null. assertNull(result.get(1)); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/FieldAssertionMapperTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/FieldAssertionMapperTest.java new file mode 100644 index 0000000000000..7758aaa986fed --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/FieldAssertionMapperTest.java @@ -0,0 +1,100 @@ +package com.linkedin.datahub.graphql.types.assertion; + +import com.linkedin.assertion.AssertionStdOperator; +import com.linkedin.assertion.FieldAssertionInfo; +import com.linkedin.assertion.FieldAssertionType; +import com.linkedin.assertion.FieldMetricAssertion; +import com.linkedin.assertion.FieldMetricType; +import com.linkedin.assertion.FieldTransform; +import com.linkedin.assertion.FieldTransformType; +import com.linkedin.assertion.FieldValuesAssertion; +import com.linkedin.assertion.FieldValuesFailThreshold; +import com.linkedin.assertion.FieldValuesFailThresholdType; +import com.linkedin.common.urn.Urn; +import com.linkedin.dataset.DatasetFilter; +import com.linkedin.dataset.DatasetFilterType; +import com.linkedin.schema.SchemaFieldSpec; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class FieldAssertionMapperTest { + @Test + public void testMapFieldValuesAssertionInfo() throws Exception { + FieldAssertionInfo fieldAssertionInfo = + new FieldAssertionInfo() + .setEntity(new Urn("urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)")) + .setFilter( + new DatasetFilter().setType(DatasetFilterType.SQL).setSql("WHERE value > 5;")) + .setType(FieldAssertionType.FIELD_VALUES) + .setFieldValuesAssertion( + new FieldValuesAssertion() + .setExcludeNulls(true) + .setFailThreshold( + new FieldValuesFailThreshold() + .setType(FieldValuesFailThresholdType.PERCENTAGE) + .setValue(5L)) + .setField( + new SchemaFieldSpec() + .setPath("path") + .setType("STRING") + .setNativeType("VARCHAR")) + .setOperator(AssertionStdOperator.IS_TRUE) + .setTransform(new FieldTransform().setType(FieldTransformType.LENGTH))); + + com.linkedin.datahub.graphql.generated.FieldAssertionInfo result = + FieldAssertionMapper.mapFieldAssertionInfo(null, fieldAssertionInfo); + Assert.assertEquals(result.getEntityUrn(), "urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)"); + Assert.assertEquals( + result.getType(), com.linkedin.datahub.graphql.generated.FieldAssertionType.FIELD_VALUES); + Assert.assertEquals( + result.getFilter().getType(), com.linkedin.datahub.graphql.generated.DatasetFilterType.SQL); + Assert.assertEquals(result.getFilter().getSql(), "WHERE value > 5;"); + Assert.assertEquals(result.getFieldValuesAssertion().getField().getPath(), "path"); + Assert.assertEquals(result.getFieldValuesAssertion().getField().getType(), "STRING"); + Assert.assertEquals(result.getFieldValuesAssertion().getField().getNativeType(), "VARCHAR"); + Assert.assertEquals( + result.getFieldValuesAssertion().getOperator(), + com.linkedin.datahub.graphql.generated.AssertionStdOperator.IS_TRUE); + Assert.assertEquals( + result.getFieldValuesAssertion().getTransform().getType(), + com.linkedin.datahub.graphql.generated.FieldTransformType.LENGTH); + Assert.assertEquals(result.getFieldValuesAssertion().getExcludeNulls(), true); + Assert.assertEquals( + result.getFieldValuesAssertion().getFailThreshold().getType(), + com.linkedin.datahub.graphql.generated.FieldValuesFailThresholdType.PERCENTAGE); + Assert.assertEquals( + result.getFieldValuesAssertion().getFailThreshold().getValue(), Long.valueOf(5L)); + } + + @Test + public void testMapFieldMetricAssertionInfo() throws Exception { + FieldAssertionInfo fieldAssertionInfo = + new FieldAssertionInfo() + .setEntity(new Urn("urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)")) + .setType(FieldAssertionType.FIELD_METRIC) + .setFieldMetricAssertion( + new FieldMetricAssertion() + .setField( + new SchemaFieldSpec() + .setPath("path") + .setType("STRING") + .setNativeType("VARCHAR")) + .setOperator(AssertionStdOperator.IS_TRUE) + .setMetric(FieldMetricType.MEDIAN)); + + com.linkedin.datahub.graphql.generated.FieldAssertionInfo result = + FieldAssertionMapper.mapFieldAssertionInfo(null, fieldAssertionInfo); + Assert.assertEquals(result.getEntityUrn(), "urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)"); + Assert.assertEquals( + result.getType(), com.linkedin.datahub.graphql.generated.FieldAssertionType.FIELD_METRIC); + Assert.assertEquals(result.getFieldMetricAssertion().getField().getPath(), "path"); + Assert.assertEquals(result.getFieldMetricAssertion().getField().getType(), "STRING"); + Assert.assertEquals(result.getFieldMetricAssertion().getField().getNativeType(), "VARCHAR"); + Assert.assertEquals( + result.getFieldMetricAssertion().getOperator(), + com.linkedin.datahub.graphql.generated.AssertionStdOperator.IS_TRUE); + Assert.assertEquals( + result.getFieldMetricAssertion().getMetric(), + com.linkedin.datahub.graphql.generated.FieldMetricType.MEDIAN); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/FreshnessAssertionMapperTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/FreshnessAssertionMapperTest.java new file mode 100644 index 0000000000000..b69ed02bdfd62 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/FreshnessAssertionMapperTest.java @@ -0,0 +1,82 @@ +package com.linkedin.datahub.graphql.types.assertion; + +import com.linkedin.assertion.FixedIntervalSchedule; +import com.linkedin.assertion.FreshnessAssertionInfo; +import com.linkedin.assertion.FreshnessAssertionSchedule; +import com.linkedin.assertion.FreshnessAssertionScheduleType; +import com.linkedin.assertion.FreshnessAssertionType; +import com.linkedin.assertion.FreshnessCronSchedule; +import com.linkedin.common.urn.Urn; +import com.linkedin.dataset.DatasetFilter; +import com.linkedin.dataset.DatasetFilterType; +import com.linkedin.timeseries.CalendarInterval; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class FreshnessAssertionMapperTest { + @Test + public void testMapCronFreshnessAssertionInfo() throws Exception { + FreshnessAssertionInfo freshnessAssertionInfo = + new FreshnessAssertionInfo() + .setEntity(new Urn("urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)")) + .setType(FreshnessAssertionType.DATASET_CHANGE) + .setFilter( + new DatasetFilter().setType(DatasetFilterType.SQL).setSql("WHERE value > 5;")) + .setSchedule( + new FreshnessAssertionSchedule() + .setType(FreshnessAssertionScheduleType.CRON) + .setCron( + new FreshnessCronSchedule() + .setCron("0 0 0 * * ? *") + .setTimezone("America/Los_Angeles") + .setWindowStartOffsetMs(10L))); + + com.linkedin.datahub.graphql.generated.FreshnessAssertionInfo result = + FreshnessAssertionMapper.mapFreshnessAssertionInfo(null, freshnessAssertionInfo); + Assert.assertEquals(result.getEntityUrn(), "urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)"); + Assert.assertEquals( + result.getType(), + com.linkedin.datahub.graphql.generated.FreshnessAssertionType.DATASET_CHANGE); + Assert.assertEquals( + result.getFilter().getType(), com.linkedin.datahub.graphql.generated.DatasetFilterType.SQL); + Assert.assertEquals(result.getFilter().getSql(), "WHERE value > 5;"); + Assert.assertEquals( + result.getSchedule().getType(), + com.linkedin.datahub.graphql.generated.FreshnessAssertionScheduleType.CRON); + Assert.assertEquals(result.getSchedule().getCron().getCron(), "0 0 0 * * ? *"); + Assert.assertEquals(result.getSchedule().getCron().getTimezone(), "America/Los_Angeles"); + Assert.assertEquals(result.getSchedule().getCron().getWindowStartOffsetMs(), Long.valueOf(10L)); + } + + @Test + public void testMapFixedIntervalFreshnessAssertionInfo() throws Exception { + FreshnessAssertionInfo freshnessAssertionInfo = + new FreshnessAssertionInfo() + .setEntity(new Urn("urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)")) + .setType(FreshnessAssertionType.DATASET_CHANGE) + .setFilter( + new DatasetFilter().setType(DatasetFilterType.SQL).setSql("WHERE value > 5;")) + .setSchedule( + new FreshnessAssertionSchedule() + .setType(FreshnessAssertionScheduleType.FIXED_INTERVAL) + .setFixedInterval( + new FixedIntervalSchedule().setUnit(CalendarInterval.DAY).setMultiple(10))); + + com.linkedin.datahub.graphql.generated.FreshnessAssertionInfo result = + FreshnessAssertionMapper.mapFreshnessAssertionInfo(null, freshnessAssertionInfo); + Assert.assertEquals(result.getEntityUrn(), "urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)"); + Assert.assertEquals( + result.getType(), + com.linkedin.datahub.graphql.generated.FreshnessAssertionType.DATASET_CHANGE); + Assert.assertEquals( + result.getFilter().getType(), com.linkedin.datahub.graphql.generated.DatasetFilterType.SQL); + Assert.assertEquals(result.getFilter().getSql(), "WHERE value > 5;"); + Assert.assertEquals( + result.getSchedule().getType(), + com.linkedin.datahub.graphql.generated.FreshnessAssertionScheduleType.FIXED_INTERVAL); + Assert.assertEquals( + result.getSchedule().getFixedInterval().getUnit(), + com.linkedin.datahub.graphql.generated.DateInterval.DAY); + Assert.assertEquals(result.getSchedule().getFixedInterval().getMultiple(), 10); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/SqlAssertionMapperTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/SqlAssertionMapperTest.java new file mode 100644 index 0000000000000..271362c9fd846 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/SqlAssertionMapperTest.java @@ -0,0 +1,78 @@ +package com.linkedin.datahub.graphql.types.assertion; + +import com.linkedin.assertion.AssertionStdOperator; +import com.linkedin.assertion.AssertionStdParameter; +import com.linkedin.assertion.AssertionStdParameterType; +import com.linkedin.assertion.AssertionStdParameters; +import com.linkedin.assertion.AssertionValueChangeType; +import com.linkedin.assertion.SqlAssertionInfo; +import com.linkedin.assertion.SqlAssertionType; +import com.linkedin.common.urn.Urn; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class SqlAssertionMapperTest { + @Test + public void testMapMetricSqlAssertionInfo() throws Exception { + SqlAssertionInfo sqlAssertionInfo = + new SqlAssertionInfo() + .setEntity(new Urn("urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)")) + .setType(SqlAssertionType.METRIC) + .setStatement("SELECT COUNT(*) FROM foo.bar.baz") + .setOperator(AssertionStdOperator.GREATER_THAN) + .setParameters( + new AssertionStdParameters() + .setValue( + new AssertionStdParameter() + .setType(AssertionStdParameterType.NUMBER) + .setValue(("5")))); + + com.linkedin.datahub.graphql.generated.SqlAssertionInfo result = + SqlAssertionMapper.mapSqlAssertionInfo(sqlAssertionInfo); + Assert.assertEquals(result.getEntityUrn(), "urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)"); + Assert.assertEquals( + result.getType(), com.linkedin.datahub.graphql.generated.SqlAssertionType.METRIC); + Assert.assertEquals(result.getStatement(), "SELECT COUNT(*) FROM foo.bar.baz"); + Assert.assertEquals( + result.getOperator(), + com.linkedin.datahub.graphql.generated.AssertionStdOperator.GREATER_THAN); + Assert.assertEquals( + result.getParameters().getValue().getType(), + com.linkedin.datahub.graphql.generated.AssertionStdParameterType.NUMBER); + Assert.assertEquals(result.getParameters().getValue().getValue(), "5"); + } + + @Test + public void testMapMetricChangeSqlAssertionInfo() throws Exception { + SqlAssertionInfo sqlAssertionInfo = + new SqlAssertionInfo() + .setEntity(new Urn("urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)")) + .setType(SqlAssertionType.METRIC_CHANGE) + .setStatement("SELECT COUNT(*) FROM foo.bar.baz") + .setChangeType(AssertionValueChangeType.ABSOLUTE) + .setOperator(AssertionStdOperator.GREATER_THAN) + .setParameters( + new AssertionStdParameters() + .setValue( + new AssertionStdParameter() + .setType(AssertionStdParameterType.NUMBER) + .setValue(("5")))); + + com.linkedin.datahub.graphql.generated.SqlAssertionInfo result = + SqlAssertionMapper.mapSqlAssertionInfo(sqlAssertionInfo); + Assert.assertEquals(result.getEntityUrn(), "urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)"); + Assert.assertEquals( + result.getType(), com.linkedin.datahub.graphql.generated.SqlAssertionType.METRIC_CHANGE); + Assert.assertEquals(result.getStatement(), "SELECT COUNT(*) FROM foo.bar.baz"); + Assert.assertEquals( + result.getOperator(), + com.linkedin.datahub.graphql.generated.AssertionStdOperator.GREATER_THAN); + Assert.assertEquals( + result.getParameters().getValue().getType(), + com.linkedin.datahub.graphql.generated.AssertionStdParameterType.NUMBER); + Assert.assertEquals(result.getParameters().getValue().getValue(), "5"); + Assert.assertEquals( + result.getChangeType(), + com.linkedin.datahub.graphql.generated.AssertionValueChangeType.ABSOLUTE); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/VolumeAssertionMapperTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/VolumeAssertionMapperTest.java new file mode 100644 index 0000000000000..f23fadb699207 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/assertion/VolumeAssertionMapperTest.java @@ -0,0 +1,207 @@ +package com.linkedin.datahub.graphql.types.assertion; + +import com.linkedin.assertion.AssertionStdOperator; +import com.linkedin.assertion.AssertionStdParameter; +import com.linkedin.assertion.AssertionStdParameterType; +import com.linkedin.assertion.AssertionStdParameters; +import com.linkedin.assertion.AssertionValueChangeType; +import com.linkedin.assertion.IncrementingSegmentFieldTransformer; +import com.linkedin.assertion.IncrementingSegmentFieldTransformerType; +import com.linkedin.assertion.IncrementingSegmentRowCountChange; +import com.linkedin.assertion.IncrementingSegmentRowCountTotal; +import com.linkedin.assertion.RowCountChange; +import com.linkedin.assertion.RowCountTotal; +import com.linkedin.assertion.VolumeAssertionInfo; +import com.linkedin.assertion.VolumeAssertionType; +import com.linkedin.common.urn.Urn; +import com.linkedin.dataset.DatasetFilter; +import com.linkedin.dataset.DatasetFilterType; +import com.linkedin.schema.SchemaFieldSpec; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class VolumeAssertionMapperTest { + @Test + public void testMapRowCountTotalVolumeAssertionInfo() throws Exception { + VolumeAssertionInfo volumeAssertionInfo = + new VolumeAssertionInfo() + .setEntity(new Urn("urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)")) + .setType(VolumeAssertionType.ROW_COUNT_TOTAL) + .setFilter( + new DatasetFilter().setType(DatasetFilterType.SQL).setSql("WHERE value > 5;")) + .setRowCountTotal( + new RowCountTotal() + .setOperator(AssertionStdOperator.GREATER_THAN_OR_EQUAL_TO) + .setParameters( + new AssertionStdParameters() + .setValue( + new AssertionStdParameter() + .setType(AssertionStdParameterType.NUMBER) + .setValue("10")))); + + com.linkedin.datahub.graphql.generated.VolumeAssertionInfo result = + VolumeAssertionMapper.mapVolumeAssertionInfo(null, volumeAssertionInfo); + Assert.assertEquals(result.getEntityUrn(), "urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)"); + Assert.assertEquals( + result.getType(), + com.linkedin.datahub.graphql.generated.VolumeAssertionType.ROW_COUNT_TOTAL); + Assert.assertEquals( + result.getFilter().getType(), com.linkedin.datahub.graphql.generated.DatasetFilterType.SQL); + Assert.assertEquals(result.getFilter().getSql(), "WHERE value > 5;"); + Assert.assertEquals( + result.getRowCountTotal().getOperator(), + com.linkedin.datahub.graphql.generated.AssertionStdOperator.GREATER_THAN_OR_EQUAL_TO); + Assert.assertEquals( + result.getRowCountTotal().getParameters().getValue().getType(), + com.linkedin.datahub.graphql.generated.AssertionStdParameterType.NUMBER); + Assert.assertEquals(result.getRowCountTotal().getParameters().getValue().getValue(), "10"); + } + + @Test + public void testMapRowCountChangeVolumeAssertionInfo() throws Exception { + VolumeAssertionInfo volumeAssertionInfo = + new VolumeAssertionInfo() + .setEntity(new Urn("urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)")) + .setType(VolumeAssertionType.ROW_COUNT_CHANGE) + .setFilter( + new DatasetFilter().setType(DatasetFilterType.SQL).setSql("WHERE value > 5;")) + .setRowCountChange( + new RowCountChange() + .setOperator(AssertionStdOperator.GREATER_THAN_OR_EQUAL_TO) + .setParameters( + new AssertionStdParameters() + .setValue( + new AssertionStdParameter() + .setType(AssertionStdParameterType.NUMBER) + .setValue("10"))) + .setType(AssertionValueChangeType.ABSOLUTE)); + + com.linkedin.datahub.graphql.generated.VolumeAssertionInfo result = + VolumeAssertionMapper.mapVolumeAssertionInfo(null, volumeAssertionInfo); + Assert.assertEquals(result.getEntityUrn(), "urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)"); + Assert.assertEquals( + result.getType(), + com.linkedin.datahub.graphql.generated.VolumeAssertionType.ROW_COUNT_CHANGE); + Assert.assertEquals( + result.getFilter().getType(), com.linkedin.datahub.graphql.generated.DatasetFilterType.SQL); + Assert.assertEquals(result.getFilter().getSql(), "WHERE value > 5;"); + Assert.assertEquals( + result.getRowCountChange().getOperator(), + com.linkedin.datahub.graphql.generated.AssertionStdOperator.GREATER_THAN_OR_EQUAL_TO); + Assert.assertEquals( + result.getRowCountChange().getParameters().getValue().getType(), + com.linkedin.datahub.graphql.generated.AssertionStdParameterType.NUMBER); + Assert.assertEquals(result.getRowCountChange().getParameters().getValue().getValue(), "10"); + Assert.assertEquals( + result.getRowCountChange().getType(), + com.linkedin.datahub.graphql.generated.AssertionValueChangeType.ABSOLUTE); + } + + @Test + public void testMapIncrementingSegmentRowCountTotalVolumeAssertionInfo() throws Exception { + VolumeAssertionInfo volumeAssertionInfo = + new VolumeAssertionInfo() + .setEntity(new Urn("urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)")) + .setType(VolumeAssertionType.INCREMENTING_SEGMENT_ROW_COUNT_TOTAL) + .setIncrementingSegmentRowCountTotal( + new IncrementingSegmentRowCountTotal() + .setOperator(AssertionStdOperator.GREATER_THAN_OR_EQUAL_TO) + .setParameters( + new AssertionStdParameters() + .setValue( + new AssertionStdParameter() + .setType(AssertionStdParameterType.NUMBER) + .setValue("10"))) + .setSegment( + new com.linkedin.assertion.IncrementingSegmentSpec() + .setField( + new SchemaFieldSpec() + .setPath("path") + .setNativeType("VARCHAR") + .setType("STRING")) + .setTransformer( + new IncrementingSegmentFieldTransformer() + .setType(IncrementingSegmentFieldTransformerType.CEILING) + .setNativeType("CEILING")))); + + com.linkedin.datahub.graphql.generated.VolumeAssertionInfo result = + VolumeAssertionMapper.mapVolumeAssertionInfo(null, volumeAssertionInfo); + Assert.assertEquals(result.getEntityUrn(), "urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)"); + Assert.assertEquals( + result.getType(), + com.linkedin.datahub.graphql.generated.VolumeAssertionType + .INCREMENTING_SEGMENT_ROW_COUNT_TOTAL); + Assert.assertEquals( + result.getIncrementingSegmentRowCountTotal().getOperator(), + com.linkedin.datahub.graphql.generated.AssertionStdOperator.GREATER_THAN_OR_EQUAL_TO); + Assert.assertEquals( + result.getIncrementingSegmentRowCountTotal().getParameters().getValue().getType(), + com.linkedin.datahub.graphql.generated.AssertionStdParameterType.NUMBER); + Assert.assertEquals( + result.getIncrementingSegmentRowCountTotal().getParameters().getValue().getValue(), "10"); + Assert.assertEquals( + result.getIncrementingSegmentRowCountTotal().getSegment().getField().getPath(), "path"); + Assert.assertEquals( + result.getIncrementingSegmentRowCountTotal().getSegment().getField().getNativeType(), + "VARCHAR"); + Assert.assertEquals( + result.getIncrementingSegmentRowCountTotal().getSegment().getField().getType(), "STRING"); + Assert.assertEquals( + result.getIncrementingSegmentRowCountTotal().getSegment().getTransformer().getType(), + com.linkedin.datahub.graphql.generated.IncrementingSegmentFieldTransformerType.CEILING); + Assert.assertEquals( + result.getIncrementingSegmentRowCountTotal().getSegment().getTransformer().getNativeType(), + "CEILING"); + } + + @Test + public void testMapIncrementingSegmentRowCountChangeVolumeAssertionInfo() throws Exception { + VolumeAssertionInfo volumeAssertionInfo = + new VolumeAssertionInfo() + .setEntity(new Urn("urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)")) + .setType(VolumeAssertionType.INCREMENTING_SEGMENT_ROW_COUNT_CHANGE) + .setIncrementingSegmentRowCountChange( + new IncrementingSegmentRowCountChange() + .setType(AssertionValueChangeType.ABSOLUTE) + .setOperator(AssertionStdOperator.GREATER_THAN_OR_EQUAL_TO) + .setParameters( + new AssertionStdParameters() + .setValue( + new AssertionStdParameter() + .setType(AssertionStdParameterType.NUMBER) + .setValue("10"))) + .setSegment( + new com.linkedin.assertion.IncrementingSegmentSpec() + .setField( + new SchemaFieldSpec() + .setPath("path") + .setNativeType("VARCHAR") + .setType("STRING")))); + + com.linkedin.datahub.graphql.generated.VolumeAssertionInfo result = + VolumeAssertionMapper.mapVolumeAssertionInfo(null, volumeAssertionInfo); + Assert.assertEquals(result.getEntityUrn(), "urn:li:dataset:(urn:li:dataPlatform:foo,bar,baz)"); + Assert.assertEquals( + result.getType(), + com.linkedin.datahub.graphql.generated.VolumeAssertionType + .INCREMENTING_SEGMENT_ROW_COUNT_CHANGE); + Assert.assertEquals( + result.getIncrementingSegmentRowCountChange().getType(), + com.linkedin.datahub.graphql.generated.AssertionValueChangeType.ABSOLUTE); + Assert.assertEquals( + result.getIncrementingSegmentRowCountChange().getOperator(), + com.linkedin.datahub.graphql.generated.AssertionStdOperator.GREATER_THAN_OR_EQUAL_TO); + Assert.assertEquals( + result.getIncrementingSegmentRowCountChange().getParameters().getValue().getType(), + com.linkedin.datahub.graphql.generated.AssertionStdParameterType.NUMBER); + Assert.assertEquals( + result.getIncrementingSegmentRowCountChange().getParameters().getValue().getValue(), "10"); + Assert.assertEquals( + result.getIncrementingSegmentRowCountChange().getSegment().getField().getPath(), "path"); + Assert.assertEquals( + result.getIncrementingSegmentRowCountChange().getSegment().getField().getNativeType(), + "VARCHAR"); + Assert.assertEquals( + result.getIncrementingSegmentRowCountChange().getSegment().getField().getType(), "STRING"); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/datacontract/DataContractMapperTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/datacontract/DataContractMapperTest.java new file mode 100644 index 0000000000000..49f5a985ea4a3 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/datacontract/DataContractMapperTest.java @@ -0,0 +1,180 @@ +package com.linkedin.datahub.graphql.types.datacontract; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.StringMap; +import com.linkedin.datacontract.DataContractProperties; +import com.linkedin.datacontract.DataContractState; +import com.linkedin.datacontract.DataContractStatus; +import com.linkedin.datacontract.DataQualityContract; +import com.linkedin.datacontract.DataQualityContractArray; +import com.linkedin.datacontract.FreshnessContract; +import com.linkedin.datacontract.FreshnessContractArray; +import com.linkedin.datacontract.SchemaContract; +import com.linkedin.datacontract.SchemaContractArray; +import com.linkedin.datahub.graphql.generated.DataContract; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.metadata.Constants; +import java.util.Collections; +import org.testng.annotations.Test; + +public class DataContractMapperTest { + + @Test + public void testMapAllFields() throws Exception { + EntityResponse entityResponse = new EntityResponse(); + Urn urn = Urn.createFromString("urn:li:dataContract:1"); + Urn dataQualityAssertionUrn = Urn.createFromString("urn:li:assertion:quality"); + Urn dataQualityAssertionUrn2 = Urn.createFromString("urn:li:assertion:quality2"); + + Urn freshnessAssertionUrn = Urn.createFromString("urn:li:assertion:freshness"); + Urn schemaAssertionUrn = Urn.createFromString("urn:li:assertion:schema"); + Urn datasetUrn = + Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:snowflake,test,PROD)"); + entityResponse.setUrn(urn); + + EnvelopedAspect envelopedDataContractProperties = new EnvelopedAspect(); + DataContractProperties dataContractProperties = new DataContractProperties(); + dataContractProperties.setDataQuality( + new DataQualityContractArray( + ImmutableList.of( + new DataQualityContract().setAssertion(dataQualityAssertionUrn), + new DataQualityContract().setAssertion(dataQualityAssertionUrn2)))); + dataContractProperties.setFreshness( + new FreshnessContractArray( + ImmutableList.of(new FreshnessContract().setAssertion(freshnessAssertionUrn)))); + dataContractProperties.setSchema( + new SchemaContractArray( + ImmutableList.of(new SchemaContract().setAssertion(schemaAssertionUrn)))); + + dataContractProperties.setEntity(datasetUrn); + + envelopedDataContractProperties.setValue(new Aspect(dataContractProperties.data())); + + EnvelopedAspect envelopedDataContractStatus = new EnvelopedAspect(); + DataContractStatus status = new DataContractStatus(); + status.setState(DataContractState.PENDING); + status.setCustomProperties(new StringMap(ImmutableMap.of("key", "value"))); + + envelopedDataContractStatus.setValue(new Aspect(status.data())); + entityResponse.setAspects( + new EnvelopedAspectMap( + Collections.singletonMap( + Constants.DATA_CONTRACT_PROPERTIES_ASPECT_NAME, envelopedDataContractProperties))); + + entityResponse.setAspects( + new EnvelopedAspectMap( + ImmutableMap.of( + Constants.DATA_CONTRACT_PROPERTIES_ASPECT_NAME, + envelopedDataContractProperties, + Constants.DATA_CONTRACT_STATUS_ASPECT_NAME, + envelopedDataContractStatus))); + + DataContract dataContract = DataContractMapper.mapContract(entityResponse); + assertNotNull(dataContract); + assertEquals(dataContract.getUrn(), urn.toString()); + assertEquals(dataContract.getType(), EntityType.DATA_CONTRACT); + assertEquals( + dataContract.getStatus().getState(), + com.linkedin.datahub.graphql.generated.DataContractState.PENDING); + assertEquals(dataContract.getProperties().getEntityUrn(), datasetUrn.toString()); + assertEquals(dataContract.getProperties().getDataQuality().size(), 2); + assertEquals( + dataContract.getProperties().getDataQuality().get(0).getAssertion().getUrn(), + dataQualityAssertionUrn.toString()); + assertEquals( + dataContract.getProperties().getDataQuality().get(1).getAssertion().getUrn(), + dataQualityAssertionUrn2.toString()); + assertEquals(dataContract.getProperties().getFreshness().size(), 1); + assertEquals( + dataContract.getProperties().getFreshness().get(0).getAssertion().getUrn(), + freshnessAssertionUrn.toString()); + assertEquals(dataContract.getProperties().getSchema().size(), 1); + assertEquals( + dataContract.getProperties().getSchema().get(0).getAssertion().getUrn(), + schemaAssertionUrn.toString()); + } + + @Test + public void testMapRequiredFields() throws Exception { + EntityResponse entityResponse = new EntityResponse(); + Urn urn = Urn.createFromString("urn:li:dataContract:1"); + Urn datasetUrn = + Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:snowflake,test,PROD)"); + entityResponse.setUrn(urn); + + EnvelopedAspect envelopedDataContractProperties = new EnvelopedAspect(); + DataContractProperties dataContractProperties = new DataContractProperties(); + dataContractProperties.setEntity(datasetUrn); + envelopedDataContractProperties.setValue(new Aspect(dataContractProperties.data())); + + EnvelopedAspect envelopedDataContractStatus = new EnvelopedAspect(); + DataContractStatus status = new DataContractStatus(); + status.setState(DataContractState.PENDING); + status.setCustomProperties(new StringMap(ImmutableMap.of("key", "value"))); + + envelopedDataContractStatus.setValue(new Aspect(status.data())); + entityResponse.setAspects( + new EnvelopedAspectMap( + Collections.singletonMap( + Constants.DATA_CONTRACT_PROPERTIES_ASPECT_NAME, envelopedDataContractProperties))); + + entityResponse.setAspects( + new EnvelopedAspectMap( + ImmutableMap.of( + Constants.DATA_CONTRACT_PROPERTIES_ASPECT_NAME, + envelopedDataContractProperties, + Constants.DATA_CONTRACT_STATUS_ASPECT_NAME, + envelopedDataContractStatus))); + + DataContract dataContract = DataContractMapper.mapContract(entityResponse); + assertNotNull(dataContract); + assertEquals(dataContract.getUrn(), urn.toString()); + assertEquals(dataContract.getType(), EntityType.DATA_CONTRACT); + assertEquals( + dataContract.getStatus().getState(), + com.linkedin.datahub.graphql.generated.DataContractState.PENDING); + assertEquals(dataContract.getProperties().getEntityUrn(), datasetUrn.toString()); + assertNull(dataContract.getProperties().getDataQuality()); + assertNull(dataContract.getProperties().getSchema()); + assertNull(dataContract.getProperties().getFreshness()); + } + + @Test + public void testMapNoStatus() throws Exception { + EntityResponse entityResponse = new EntityResponse(); + Urn urn = Urn.createFromString("urn:li:dataContract:1"); + Urn datasetUrn = + Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:snowflake,test,PROD)"); + entityResponse.setUrn(urn); + + EnvelopedAspect envelopedDataContractProperties = new EnvelopedAspect(); + DataContractProperties dataContractProperties = new DataContractProperties(); + dataContractProperties.setEntity(datasetUrn); + envelopedDataContractProperties.setValue(new Aspect(dataContractProperties.data())); + + entityResponse.setAspects( + new EnvelopedAspectMap( + ImmutableMap.of( + Constants.DATA_CONTRACT_PROPERTIES_ASPECT_NAME, envelopedDataContractProperties))); + + DataContract dataContract = DataContractMapper.mapContract(entityResponse); + assertNotNull(dataContract); + assertEquals(dataContract.getUrn(), urn.toString()); + assertEquals(dataContract.getType(), EntityType.DATA_CONTRACT); + assertNull(dataContract.getStatus()); + assertEquals(dataContract.getProperties().getEntityUrn(), datasetUrn.toString()); + assertNull(dataContract.getProperties().getDataQuality()); + assertNull(dataContract.getProperties().getSchema()); + assertNull(dataContract.getProperties().getFreshness()); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/datacontract/DataContractTypeTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/datacontract/DataContractTypeTest.java new file mode 100644 index 0000000000000..241775e5ab48c --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/datacontract/DataContractTypeTest.java @@ -0,0 +1,152 @@ +package com.linkedin.datahub.graphql.types.datacontract; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.nullable; +import static org.testng.Assert.*; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datacontract.DataContractProperties; +import com.linkedin.datacontract.DataContractState; +import com.linkedin.datacontract.DataContractStatus; +import com.linkedin.datacontract.DataQualityContract; +import com.linkedin.datacontract.DataQualityContractArray; +import com.linkedin.datacontract.FreshnessContract; +import com.linkedin.datacontract.FreshnessContractArray; +import com.linkedin.datacontract.SchemaContract; +import com.linkedin.datacontract.SchemaContractArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DataContract; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.key.DataContractKey; +import com.linkedin.r2.RemoteInvocationException; +import graphql.execution.DataFetcherResult; +import io.datahubproject.metadata.context.OperationContext; +import java.util.HashSet; +import java.util.List; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class DataContractTypeTest { + + private static final Urn TEST_DATASET_URN = + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:snowflake,test,PROD)"); + private static final Urn DATA_QUALITY_ASSERTION_URN = UrnUtils.getUrn("urn:li:assertion:quality"); + private static final Urn FRESHNESS_ASSERTION_URN = UrnUtils.getUrn("urn:li:assertion:freshness"); + private static final Urn SCHEMA_ASSERTION_URN = UrnUtils.getUrn("urn:li:assertion:schema"); + private static final String TEST_DATA_CONTRACT_1_URN = "urn:li:dataContract:id-1"; + private static final DataContractKey TEST_DATA_CONTRACT_1_KEY = + new DataContractKey().setId("id-1"); + private static final DataContractProperties TEST_DATA_CONTRACT_1_PROPERTIES = + new DataContractProperties() + .setEntity(TEST_DATASET_URN) + .setDataQuality( + new DataQualityContractArray( + ImmutableList.of( + new DataQualityContract().setAssertion(DATA_QUALITY_ASSERTION_URN)))) + .setFreshness( + new FreshnessContractArray( + ImmutableList.of(new FreshnessContract().setAssertion(FRESHNESS_ASSERTION_URN)))) + .setSchema( + new SchemaContractArray( + ImmutableList.of(new SchemaContract().setAssertion(SCHEMA_ASSERTION_URN)))); + private static final DataContractStatus TEST_DATA_CONTRACT_1_STATUS = + new DataContractStatus().setState(DataContractState.ACTIVE); + + private static final String TEST_DATA_CONTRACT_2_URN = "urn:li:dataContract:id-2"; + + @Test + public void testBatchLoad() throws Exception { + + EntityClient client = Mockito.mock(EntityClient.class); + + Urn dataContractUrn1 = Urn.createFromString(TEST_DATA_CONTRACT_1_URN); + Urn dataContractUrn2 = Urn.createFromString(TEST_DATA_CONTRACT_2_URN); + + Mockito.when( + client.batchGetV2( + any(OperationContext.class), + Mockito.eq(Constants.DATA_CONTRACT_ENTITY_NAME), + Mockito.eq(new HashSet<>(ImmutableSet.of(dataContractUrn1, dataContractUrn2))), + Mockito.eq(DataContractType.ASPECTS_TO_FETCH))) + .thenReturn( + ImmutableMap.of( + dataContractUrn1, + new EntityResponse() + .setEntityName(Constants.DATA_CONTRACT_ENTITY_NAME) + .setUrn(dataContractUrn1) + .setAspects( + new EnvelopedAspectMap( + ImmutableMap.of( + Constants.DATA_CONTRACT_KEY_ASPECT_NAME, + new EnvelopedAspect() + .setValue(new Aspect(TEST_DATA_CONTRACT_1_KEY.data())), + Constants.DATA_CONTRACT_PROPERTIES_ASPECT_NAME, + new EnvelopedAspect() + .setValue(new Aspect(TEST_DATA_CONTRACT_1_PROPERTIES.data())), + Constants.DATA_CONTRACT_STATUS_ASPECT_NAME, + new EnvelopedAspect() + .setValue(new Aspect(TEST_DATA_CONTRACT_1_STATUS.data()))))))); + + DataContractType type = new DataContractType(client); + + QueryContext mockContext = getMockAllowContext(); + List> result = + type.batchLoad( + ImmutableList.of(TEST_DATA_CONTRACT_1_URN, TEST_DATA_CONTRACT_2_URN), mockContext); + + // Verify response + Mockito.verify(client, Mockito.times(1)) + .batchGetV2( + any(OperationContext.class), + Mockito.eq(Constants.DATA_CONTRACT_ENTITY_NAME), + Mockito.eq(ImmutableSet.of(dataContractUrn1, dataContractUrn2)), + Mockito.eq(DataContractType.ASPECTS_TO_FETCH)); + + assertEquals(result.size(), 2); + + DataContract dataContract1 = result.get(0).getData(); + assertEquals(dataContract1.getUrn(), TEST_DATA_CONTRACT_1_URN); + assertEquals(dataContract1.getType(), EntityType.DATA_CONTRACT); + assertEquals(dataContract1.getProperties().getEntityUrn(), TEST_DATASET_URN.toString()); + assertEquals(dataContract1.getProperties().getDataQuality().size(), 1); + assertEquals(dataContract1.getProperties().getSchema().size(), 1); + assertEquals(dataContract1.getProperties().getFreshness().size(), 1); + + // Assert second element is null. + assertNull(result.get(1)); + } + + @Test + public void testBatchLoadClientException() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.doThrow(RemoteInvocationException.class) + .when(mockClient) + .batchGetV2( + nullable(OperationContext.class), + Mockito.anyString(), + Mockito.anySet(), + Mockito.anySet()); + DataContractType type = new DataContractType(mockClient); + + // Execute Batch load + QueryContext context = Mockito.mock(QueryContext.class); + Mockito.when(context.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + assertThrows( + RuntimeException.class, + () -> + type.batchLoad( + ImmutableList.of(TEST_DATA_CONTRACT_1_URN, TEST_DATA_CONTRACT_2_URN), context)); + } +} diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle index 9108f3009b4ba..304bf3a67a5b2 100644 --- a/datahub-upgrade/build.gradle +++ b/datahub-upgrade/build.gradle @@ -46,6 +46,9 @@ dependencies { implementation(externalDependency.guava) { because("CVE-2023-2976") } + implementation('io.airlift:aircompressor:0.27') { + because("CVE-2024-36114") + } } @@ -91,6 +94,27 @@ bootJar { archiveFileName = "${project.name}.jar" } +bootRun { + environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml" + environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true" + environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true" + environment "SERVER_PORT", "8083" + args += ["-u", "SystemUpdate"] +} + +/** + * Runs SystemUpdate on locally running system + */ +task run(type: Exec) { + dependsOn bootJar + group = "Execution" + description = "Run the datahub-upgrade SystemUpdate process locally." + environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml" + environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true" + environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true" + commandLine "java", "-jar", "-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdate" +} + docker { name "${docker_registry}/${docker_repo}:v${version}" version "v${version}" diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BuildIndicesConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BuildIndicesConfig.java index 3510fa513b3b9..e0de8a7255d61 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BuildIndicesConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BuildIndicesConfig.java @@ -6,7 +6,6 @@ import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.graph.GraphService; -import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.systemmetadata.SystemMetadataService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; @@ -26,8 +25,7 @@ public BlockingSystemUpgrade buildIndices( final BaseElasticSearchComponentsFactory.BaseElasticSearchComponents baseElasticSearchComponents, final ConfigurationProvider configurationProvider, - final AspectDao aspectDao, - final EntityRegistry entityRegistry) { + final AspectDao aspectDao) { return new BuildIndices( systemMetadataService, @@ -36,7 +34,6 @@ public BlockingSystemUpgrade buildIndices( graphService, baseElasticSearchComponents, configurationProvider, - aspectDao, - entityRegistry); + aspectDao); } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/CleanIndicesConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/CleanIndicesConfig.java index 4f54b01459625..7559aaf3f3cdb 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/CleanIndicesConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/CleanIndicesConfig.java @@ -4,6 +4,7 @@ import com.linkedin.datahub.upgrade.system.elasticsearch.CleanIndices; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory; +import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.systemmetadata.SystemMetadataService; @@ -23,7 +24,8 @@ public NonBlockingSystemUpgrade cleanIndices( final GraphService graphService, final BaseElasticSearchComponentsFactory.BaseElasticSearchComponents baseElasticSearchComponents, - final ConfigurationProvider configurationProvider) { + final ConfigurationProvider configurationProvider, + final AspectDao aspectDao) { return new CleanIndices( systemMetadataService, @@ -31,6 +33,7 @@ public NonBlockingSystemUpgrade cleanIndices( entitySearchService, graphService, baseElasticSearchComponents, - configurationProvider); + configurationProvider, + aspectDao); } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/BuildIndices.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/BuildIndices.java index fea0479876a2e..a91bba2fa0a97 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/BuildIndices.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/BuildIndices.java @@ -1,5 +1,13 @@ package com.linkedin.datahub.upgrade.system.elasticsearch; +import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME; + +import com.datahub.util.RecordUtils; +import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.system.BlockingSystemUpgrade; import com.linkedin.datahub.upgrade.system.elasticsearch.steps.BuildIndicesPostStep; @@ -8,14 +16,17 @@ import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory; import com.linkedin.metadata.entity.AspectDao; +import com.linkedin.metadata.entity.EntityAspect; import com.linkedin.metadata.graph.GraphService; -import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.metadata.systemmetadata.SystemMetadataService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.util.Pair; import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -31,8 +42,7 @@ public BuildIndices( final BaseElasticSearchComponentsFactory.BaseElasticSearchComponents baseElasticSearchComponents, final ConfigurationProvider configurationProvider, - final AspectDao aspectDao, - final EntityRegistry entityRegistry) { + final AspectDao aspectDao) { List indexedServices = Stream.of(graphService, entitySearchService, systemMetadataService, timeseriesAspectService) @@ -41,12 +51,7 @@ public BuildIndices( .collect(Collectors.toList()); _steps = - buildSteps( - indexedServices, - baseElasticSearchComponents, - configurationProvider, - aspectDao, - entityRegistry); + buildSteps(indexedServices, baseElasticSearchComponents, configurationProvider, aspectDao); } @Override @@ -64,8 +69,14 @@ private List buildSteps( final BaseElasticSearchComponentsFactory.BaseElasticSearchComponents baseElasticSearchComponents, final ConfigurationProvider configurationProvider, - final AspectDao aspectDao, - final EntityRegistry entityRegistry) { + final AspectDao aspectDao) { + + final Set> structuredProperties; + if (configurationProvider.getStructuredProperties().isSystemUpdateEnabled()) { + structuredProperties = getActiveStructuredPropertiesDefinitions(aspectDao); + } else { + structuredProperties = Set.of(); + } final List steps = new ArrayList<>(); // Disable ES write mode/change refresh rate and clone indices @@ -74,13 +85,47 @@ private List buildSteps( baseElasticSearchComponents, indexedServices, configurationProvider, - aspectDao, - entityRegistry)); + structuredProperties)); // Configure graphService, entitySearchService, systemMetadataService, timeseriesAspectService - steps.add(new BuildIndicesStep(indexedServices)); + steps.add(new BuildIndicesStep(indexedServices, structuredProperties)); // Reset configuration (and delete clones? Or just do this regularly? Or delete clone in // pre-configure step if it already exists? - steps.add(new BuildIndicesPostStep(baseElasticSearchComponents, indexedServices)); + steps.add( + new BuildIndicesPostStep( + baseElasticSearchComponents, indexedServices, structuredProperties)); return steps; } + + static Set> getActiveStructuredPropertiesDefinitions( + AspectDao aspectDao) { + Set removedStructuredPropertyUrns; + try (Stream stream = + aspectDao.streamAspects(STRUCTURED_PROPERTY_ENTITY_NAME, STATUS_ASPECT_NAME)) { + removedStructuredPropertyUrns = + stream + .map( + entityAspect -> + Pair.of( + entityAspect.getUrn(), + RecordUtils.toRecordTemplate(Status.class, entityAspect.getMetadata()))) + .filter(status -> status.getSecond().isRemoved()) + .map(Pair::getFirst) + .collect(Collectors.toSet()); + } + + try (Stream stream = + aspectDao.streamAspects( + STRUCTURED_PROPERTY_ENTITY_NAME, STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)) { + return stream + .map( + entityAspect -> + Pair.of( + UrnUtils.getUrn(entityAspect.getUrn()), + RecordUtils.toRecordTemplate( + StructuredPropertyDefinition.class, entityAspect.getMetadata()))) + .filter( + definition -> !removedStructuredPropertyUrns.contains(definition.getKey().toString())) + .collect(Collectors.toSet()); + } + } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/CleanIndices.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/CleanIndices.java index e316481e2b07e..96aea906b021e 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/CleanIndices.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/CleanIndices.java @@ -1,16 +1,23 @@ package com.linkedin.datahub.upgrade.system.elasticsearch; +import static com.linkedin.datahub.upgrade.system.elasticsearch.BuildIndices.getActiveStructuredPropertiesDefinitions; + +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; import com.linkedin.datahub.upgrade.system.elasticsearch.steps.CleanIndicesStep; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory; +import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.metadata.systemmetadata.SystemMetadataService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.util.Pair; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; @@ -26,7 +33,15 @@ public CleanIndices( final GraphService graphService, final BaseElasticSearchComponentsFactory.BaseElasticSearchComponents baseElasticSearchComponents, - final ConfigurationProvider configurationProvider) { + final ConfigurationProvider configurationProvider, + final AspectDao aspectDao) { + + final Set> structuredProperties; + if (configurationProvider.getStructuredProperties().isSystemUpdateEnabled()) { + structuredProperties = getActiveStructuredPropertiesDefinitions(aspectDao); + } else { + structuredProperties = Set.of(); + } List indexedServices = Stream.of(graphService, entitySearchService, systemMetadataService, timeseriesAspectService) @@ -39,7 +54,8 @@ public CleanIndices( new CleanIndicesStep( baseElasticSearchComponents.getSearchClient(), configurationProvider.getElasticSearch(), - indexedServices)); + indexedServices, + structuredProperties)); } @Override diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java index a44f6d6487067..09f65c8448027 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java @@ -4,6 +4,7 @@ import static com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils.getAllReindexConfigs; import com.google.common.collect.ImmutableMap; +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.upgrade.UpgradeContext; import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; @@ -12,8 +13,11 @@ import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ReindexConfig; import com.linkedin.metadata.shared.ElasticSearchIndexed; +import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.util.Pair; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; @@ -25,8 +29,9 @@ @Slf4j public class BuildIndicesPostStep implements UpgradeStep { - private final BaseElasticSearchComponentsFactory.BaseElasticSearchComponents _esComponents; - private final List _services; + private final BaseElasticSearchComponentsFactory.BaseElasticSearchComponents esComponents; + private final List services; + private final Set> structuredProperties; @Override public String id() { @@ -44,7 +49,7 @@ public Function executable() { try { List indexConfigs = - getAllReindexConfigs(_services).stream() + getAllReindexConfigs(services, structuredProperties).stream() .filter(ReindexConfig::requiresReindex) .collect(Collectors.toList()); @@ -55,7 +60,7 @@ public Function executable() { request.settings(indexSettings); boolean ack = - _esComponents + esComponents .getSearchClient() .indices() .putSettings(request, RequestOptions.DEFAULT) @@ -69,7 +74,7 @@ public Function executable() { if (ack) { ack = IndexUtils.validateWriteBlock( - _esComponents.getSearchClient(), indexConfig.name(), false); + esComponents.getSearchClient(), indexConfig.name(), false); log.info( "Validated index {} with new settings. Settings: {}, Acknowledged: {}", indexConfig.name(), diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java index c3c9981b1dd7e..983e7f0c97f38 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java @@ -2,13 +2,9 @@ import static com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils.INDEX_BLOCKS_WRITE_SETTING; import static com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils.getAllReindexConfigs; -import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME; -import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; -import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME; -import com.datahub.util.RecordUtils; import com.google.common.collect.ImmutableMap; -import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.upgrade.UpgradeContext; import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; @@ -16,9 +12,6 @@ import com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory; -import com.linkedin.metadata.entity.AspectDao; -import com.linkedin.metadata.entity.EntityAspect; -import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ReindexConfig; import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.structured.StructuredPropertyDefinition; @@ -29,7 +22,6 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.opensearch.OpenSearchStatusException; @@ -40,11 +32,10 @@ @RequiredArgsConstructor @Slf4j public class BuildIndicesPreStep implements UpgradeStep { - private final BaseElasticSearchComponentsFactory.BaseElasticSearchComponents _esComponents; - private final List _services; - private final ConfigurationProvider _configurationProvider; - private final AspectDao _aspectDao; - private final EntityRegistry _entityRegistry; + private final BaseElasticSearchComponentsFactory.BaseElasticSearchComponents esComponents; + private final List services; + private final ConfigurationProvider configurationProvider; + private final Set> structuredProperties; @Override public String id() { @@ -60,13 +51,8 @@ public int retryCount() { public Function executable() { return (context) -> { try { - final List reindexConfigs; - if (_configurationProvider.getStructuredProperties().isSystemUpdateEnabled()) { - reindexConfigs = - getAllReindexConfigs(_services, getActiveStructuredPropertiesDefinitions(_aspectDao)); - } else { - reindexConfigs = getAllReindexConfigs(_services); - } + final List reindexConfigs = + getAllReindexConfigs(services, structuredProperties); // Get indices to update List indexConfigs = @@ -76,7 +62,7 @@ public Function executable() { for (ReindexConfig indexConfig : indexConfigs) { String indexName = - IndexUtils.resolveAlias(_esComponents.getSearchClient(), indexConfig.name()); + IndexUtils.resolveAlias(esComponents.getSearchClient(), indexConfig.name()); boolean ack = blockWrites(indexName); if (!ack) { @@ -87,11 +73,11 @@ public Function executable() { } // Clone indices - if (_configurationProvider.getElasticSearch().getBuildIndices().isCloneIndices()) { + if (configurationProvider.getElasticSearch().getBuildIndices().isCloneIndices()) { String clonedName = indexConfig.name() + "_clone_" + System.currentTimeMillis(); ResizeRequest resizeRequest = new ResizeRequest(clonedName, indexName); boolean cloneAck = - _esComponents + esComponents .getSearchClient() .indices() .clone(resizeRequest, RequestOptions.DEFAULT) @@ -121,7 +107,7 @@ private boolean blockWrites(String indexName) throws InterruptedException, IOExc boolean ack; try { ack = - _esComponents + esComponents .getSearchClient() .indices() .putSettings(request, RequestOptions.DEFAULT) @@ -145,7 +131,7 @@ private boolean blockWrites(String indexName) throws InterruptedException, IOExc } if (ack) { - ack = IndexUtils.validateWriteBlock(_esComponents.getSearchClient(), indexName, true); + ack = IndexUtils.validateWriteBlock(esComponents.getSearchClient(), indexName, true); log.info( "Validated index {} with new settings. Settings: {}, Acknowledged: {}", indexName, @@ -155,37 +141,4 @@ private boolean blockWrites(String indexName) throws InterruptedException, IOExc return ack; } - - private static Set getActiveStructuredPropertiesDefinitions( - AspectDao aspectDao) { - Set removedStructuredPropertyUrns; - try (Stream stream = - aspectDao.streamAspects(STRUCTURED_PROPERTY_ENTITY_NAME, STATUS_ASPECT_NAME)) { - removedStructuredPropertyUrns = - stream - .map( - entityAspect -> - Pair.of( - entityAspect.getUrn(), - RecordUtils.toRecordTemplate(Status.class, entityAspect.getMetadata()))) - .filter(status -> status.getSecond().isRemoved()) - .map(Pair::getFirst) - .collect(Collectors.toSet()); - } - - try (Stream stream = - aspectDao.streamAspects( - STRUCTURED_PROPERTY_ENTITY_NAME, STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)) { - return stream - .map( - entityAspect -> - Pair.of( - entityAspect.getUrn(), - RecordUtils.toRecordTemplate( - StructuredPropertyDefinition.class, entityAspect.getMetadata()))) - .filter(definition -> !removedStructuredPropertyUrns.contains(definition.getKey())) - .map(Pair::getSecond) - .collect(Collectors.toSet()); - } - } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesStep.java index d37ee173bd9af..5cf370162a312 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesStep.java @@ -1,11 +1,15 @@ package com.linkedin.datahub.upgrade.system.elasticsearch.steps; +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.upgrade.UpgradeContext; import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.metadata.shared.ElasticSearchIndexed; +import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.util.Pair; import java.util.List; +import java.util.Set; import java.util.function.Function; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -14,7 +18,8 @@ @RequiredArgsConstructor public class BuildIndicesStep implements UpgradeStep { - private final List _services; + private final List services; + private final Set> structuredProperties; @Override public String id() { @@ -30,8 +35,8 @@ public int retryCount() { public Function executable() { return (context) -> { try { - for (ElasticSearchIndexed service : _services) { - service.reindexAll(); + for (ElasticSearchIndexed service : services) { + service.reindexAll(structuredProperties); } } catch (Exception e) { log.error("BuildIndicesStep failed.", e); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java index c3a4d8ab89c07..fd5592c4ead25 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.upgrade.system.elasticsearch.steps; +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.upgrade.UpgradeContext; import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; @@ -8,7 +9,10 @@ import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.shared.ElasticSearchIndexed; +import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.util.Pair; import java.util.List; +import java.util.Set; import java.util.function.Function; import lombok.extern.slf4j.Slf4j; import org.opensearch.client.RestHighLevelClient; @@ -18,14 +22,17 @@ public class CleanIndicesStep implements UpgradeStep { private final RestHighLevelClient searchClient; private final ElasticSearchConfiguration esConfig; private final List indexedServices; + private final Set> structuredProperties; public CleanIndicesStep( final RestHighLevelClient searchClient, final ElasticSearchConfiguration esConfig, - final List indexedServices) { + final List indexedServices, + final Set> structuredProperties) { this.searchClient = searchClient; this.esConfig = esConfig; this.indexedServices = indexedServices; + this.structuredProperties = structuredProperties; } @Override @@ -42,7 +49,7 @@ public int retryCount() { public Function executable() { return (context) -> { try { - IndexUtils.getAllReindexConfigs(indexedServices) + IndexUtils.getAllReindexConfigs(indexedServices, structuredProperties) .forEach( reindexConfig -> ESIndexBuilder.cleanIndex(searchClient, esConfig, reindexConfig)); } catch (Exception e) { diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java index 52b34200991c3..99d72776ff788 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java @@ -1,8 +1,10 @@ package com.linkedin.datahub.upgrade.system.elasticsearch.util; +import com.linkedin.common.urn.Urn; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ReindexConfig; import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.util.Pair; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -27,30 +29,15 @@ private IndexUtils() {} private static List _reindexConfigs = new ArrayList<>(); - public static List getAllReindexConfigs( - List elasticSearchIndexedList) throws IOException { - // Avoid locking & reprocessing - List reindexConfigs = new ArrayList<>(_reindexConfigs); - if (reindexConfigs.isEmpty()) { - for (ElasticSearchIndexed elasticSearchIndexed : elasticSearchIndexedList) { - reindexConfigs.addAll(elasticSearchIndexed.buildReindexConfigs()); - } - _reindexConfigs = new ArrayList<>(reindexConfigs); - } - - return reindexConfigs; - } - public static List getAllReindexConfigs( List elasticSearchIndexedList, - Collection structuredProperties) + Collection> structuredProperties) throws IOException { // Avoid locking & reprocessing List reindexConfigs = new ArrayList<>(_reindexConfigs); if (reindexConfigs.isEmpty()) { for (ElasticSearchIndexed elasticSearchIndexed : elasticSearchIndexedList) { - reindexConfigs.addAll( - elasticSearchIndexed.buildReindexConfigsWithAllStructProps(structuredProperties)); + reindexConfigs.addAll(elasticSearchIndexed.buildReindexConfigs(structuredProperties)); } _reindexConfigs = new ArrayList<>(reindexConfigs); } diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json index 50a74bb0f4259..ca53932eba518 100644 --- a/datahub-web-react/package.json +++ b/datahub-web-react/package.json @@ -4,7 +4,7 @@ "private": true, "dependencies": { "@analytics/amplitude": "0.0.3", - "@analytics/google-analytics": "^0.5.2", + "@analytics/google-analytics": "^1.0.7", "@analytics/mixpanel": "^0.3.1", "@ant-design/colors": "^5.0.0", "@ant-design/icons": "^4.3.0", @@ -71,7 +71,7 @@ "react-router-dom": "^5.3", "react-syntax-highlighter": "^15.4.4", "react-visibility-sensor": "^5.1.1", - "reactour": "1.18.7", + "reactour": "^1.19.3", "remirror": "^2.0.23", "styled-components": "^5.2.1", "turndown-plugin-gfm": "^1.0.2", @@ -88,8 +88,10 @@ "build": "yarn run generate && NODE_OPTIONS='--max-old-space-size=3072 --openssl-legacy-provider' CI=false vite build", "test": "vitest", "generate": "graphql-codegen --config codegen.yml", - "lint": "eslint . --ext .ts,.tsx --quiet && yarn type-check", + "lint": "eslint . --ext .ts,.tsx --quiet && yarn format-check && yarn type-check", "lint-fix": "eslint '*/**/*.{ts,tsx}' --quiet --fix", + "format-check": "prettier --check src", + "format": "prettier --write src", "type-check": "tsc --noEmit", "type-watch": "tsc -w --noEmit" }, @@ -130,7 +132,7 @@ "less": "^4.2.0", "prettier": "^2.8.8", "source-map-explorer": "^2.5.2", - "vite": "^4.5.2", + "vite": "^4.5.3", "vite-plugin-babel-macros": "^1.0.6", "vite-plugin-static-copy": "^0.17.0", "vite-plugin-svgr": "^4.1.0", diff --git a/datahub-web-react/src/App.less b/datahub-web-react/src/App.less index 62ccac85a8fe1..5837d77a5a4e5 100644 --- a/datahub-web-react/src/App.less +++ b/datahub-web-react/src/App.less @@ -5,7 +5,7 @@ // less preprocessor configuration. @font-face { - font-family: 'Manrope'; - font-style: normal; - src: local('Manrope'), url('./fonts/manrope.woff2') format('woff2'), + font-family: 'Manrope'; + font-style: normal; + src: local('Manrope'), url('./fonts/manrope.woff2') format('woff2'); } diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx index 9f9107865aac4..aed672a34e7ca 100644 --- a/datahub-web-react/src/Mocks.tsx +++ b/datahub-web-react/src/Mocks.tsx @@ -1254,6 +1254,7 @@ export const glossaryNode5 = { export const sampleTag = { urn: 'urn:li:tag:abc-sample-tag', + type: EntityType.Tag, name: 'abc-sample-tag', description: 'sample tag description', ownership: { @@ -1453,10 +1454,10 @@ export const businessAttribute = { name: 'TestBusinessAtt-2', description: 'lorem upsum updated 12', created: { - time: 1705857132786 + time: 1705857132786, }, lastModified: { - time: 1705857132786 + time: 1705857132786, }, glossaryTerms: { terms: [ @@ -1465,10 +1466,10 @@ export const businessAttribute = { urn: 'urn:li:glossaryTerm:1', type: EntityType.GlossaryTerm, hierarchicalName: 'SampleHierarchicalName', - name: 'SampleName', + name: 'SampleName', }, - associatedUrn: 'urn:li:businessAttribute:ba1' - } + associatedUrn: 'urn:li:businessAttribute:ba1', + }, ], __typename: 'GlossaryTerms', }, @@ -1483,7 +1484,7 @@ export const businessAttribute = { name: 'abc-sample-tag', }, __typename: 'TagAssociation', - associatedUrn: 'urn:li:businessAttribute:ba1' + associatedUrn: 'urn:li:businessAttribute:ba1', }, { tag: { @@ -1493,30 +1494,30 @@ export const businessAttribute = { name: 'TestTag', }, __typename: 'TagAssociation', - associatedUrn: 'urn:li:businessAttribute:ba1' - } - ] + associatedUrn: 'urn:li:businessAttribute:ba1', + }, + ], }, customProperties: [ { key: 'prop2', value: 'val2', associatedUrn: 'urn:li:businessAttribute:ba1', - __typename: 'CustomPropertiesEntry' + __typename: 'CustomPropertiesEntry', }, { key: 'prop1', value: 'val1', associatedUrn: 'urn:li:businessAttribute:ba1', - __typename: 'CustomPropertiesEntry' + __typename: 'CustomPropertiesEntry', }, { key: 'prop3', value: 'val3', associatedUrn: 'urn:li:businessAttribute:ba1', - __typename: 'CustomPropertiesEntry' - } - ] + __typename: 'CustomPropertiesEntry', + }, + ], }, ownership: { owners: [ diff --git a/datahub-web-react/src/app/SearchRoutes.tsx b/datahub-web-react/src/app/SearchRoutes.tsx index 4ebcc6f090a4b..3343260c72bcf 100644 --- a/datahub-web-react/src/app/SearchRoutes.tsx +++ b/datahub-web-react/src/app/SearchRoutes.tsx @@ -53,15 +53,18 @@ export const SearchRoutes = (): JSX.Element => { } /> } /> } /> - { - if (!appConfigContextLoaded) { - return null; - } - if (businessAttributesFlag) { - return ; - } - return ; - }}/> + { + if (!appConfigContextLoaded) { + return null; + } + if (businessAttributesFlag) { + return ; + } + return ; + }} + /> diff --git a/datahub-web-react/src/app/analytics/README.md b/datahub-web-react/src/app/analytics/README.md index 79b82bcc2a756..881fffd59fb2c 100644 --- a/datahub-web-react/src/app/analytics/README.md +++ b/datahub-web-react/src/app/analytics/README.md @@ -48,20 +48,17 @@ const config: any = { ### Google Analytics -**Disclaimers** - -- This plugin requires use of Universal Analytics and does not yet support GA4. To create a Universal Analytics Property, follow [this guide](https://www.analyticsmania.com/other-posts/how-to-create-a-universal-analytics-property/). -- Google Analytics lacks robust support for custom event properties. For that reason many of the DataHub events discussed above will not be fully populated. Instead, we map certain fields of the DataHub event to the standard `category`, `action`, `label` fields required by GA. - 1. Open `datahub-web-react/src/conf/analytics.ts` -2. Uncomment the `googleAnalytics` field within the `config` object. -3. Replace the sample `trackingId` with the one provided by Google Analytics. +2. Uncomment the `googleAnalytics` field within the `config`. +3. Replace the sample `measurementIds` with the one provided by Google Analytics. 4. Rebuild & redeploy `datahub-frontend-react` to start tracking. +Example: + ```typescript const config: any = { googleAnalytics: { - trackingId: 'UA-24123123-01', + measurementIds: ['G-ATV123'], }, }; ``` diff --git a/datahub-web-react/src/app/analytics/plugin/googleAnalytics.ts b/datahub-web-react/src/app/analytics/plugin/googleAnalytics.ts index f60f46513272b..727258ee8d40f 100644 --- a/datahub-web-react/src/app/analytics/plugin/googleAnalytics.ts +++ b/datahub-web-react/src/app/analytics/plugin/googleAnalytics.ts @@ -2,9 +2,9 @@ import googleAnalytics from '@analytics/google-analytics'; import { Event, EventType } from '../event'; import analyticsConfig from '../../../conf/analytics'; -const gaConfigs = analyticsConfig.googleAnalytics; -const isEnabled: boolean = gaConfigs || false; -const trackingId = isEnabled ? gaConfigs.trackingId : undefined; +const ga4Configs = analyticsConfig.googleAnalytics; +const isEnabled: boolean = ga4Configs || false; +const measurementIds = isEnabled ? ga4Configs.measurementIds : undefined; const getLabelFromEvent = (event: Event) => { switch (event.type) { @@ -21,11 +21,7 @@ const getLabelFromEvent = (event: Event) => { let wrappedGoogleAnalyticsPlugin; if (isEnabled) { - /** - * Init default GA plugin - */ - const googleAnalyticsPlugin = googleAnalytics({ trackingId }); - + const googleAnalyticsPlugin = googleAnalytics({ measurementIds }); /** * Lightweight wrapper on top of the default google analytics plugin * to transform DataHub Analytics Events into the Google Analytics event diff --git a/datahub-web-react/src/app/analyticsDashboard/components/__tests__/timeSeriesChart.test.tsx b/datahub-web-react/src/app/analyticsDashboard/components/__tests__/timeSeriesChart.test.tsx index eadbb60959a0b..c528e4e627a1c 100644 --- a/datahub-web-react/src/app/analyticsDashboard/components/__tests__/timeSeriesChart.test.tsx +++ b/datahub-web-react/src/app/analyticsDashboard/components/__tests__/timeSeriesChart.test.tsx @@ -36,7 +36,7 @@ describe('timeSeriesChart', () => { }); it('compute lines works works correctly for monthly case', () => { - const chartData:TimeSeriesChart = { + const chartData: TimeSeriesChart = { title: 'Weekly Active Users', lines: [ { diff --git a/datahub-web-react/src/app/auth/ResetCredentials.tsx b/datahub-web-react/src/app/auth/ResetCredentials.tsx index 30d7f99d99d84..77f41489fcfc9 100644 --- a/datahub-web-react/src/app/auth/ResetCredentials.tsx +++ b/datahub-web-react/src/app/auth/ResetCredentials.tsx @@ -41,7 +41,9 @@ const FormInput = styled(Input)` `; const StyledFormItem = styled(Form.Item)` - .ant-input-affix-wrapper-status-error:not(.ant-input-affix-wrapper-disabled):not(.ant-input-affix-wrapper-borderless).ant-input-affix-wrapper { + .ant-input-affix-wrapper-status-error:not(.ant-input-affix-wrapper-disabled):not( + .ant-input-affix-wrapper-borderless + ).ant-input-affix-wrapper { background-color: transparent; } `; diff --git a/datahub-web-react/src/app/auth/SignUp.tsx b/datahub-web-react/src/app/auth/SignUp.tsx index e57a5930ce1ff..2eaa74946682f 100644 --- a/datahub-web-react/src/app/auth/SignUp.tsx +++ b/datahub-web-react/src/app/auth/SignUp.tsx @@ -55,7 +55,9 @@ const TitleSelector = styled(Select)` `; const StyledFormItem = styled(Form.Item)` - .ant-input-affix-wrapper-status-error:not(.ant-input-affix-wrapper-disabled):not(.ant-input-affix-wrapper-borderless).ant-input-affix-wrapper { + .ant-input-affix-wrapper-status-error:not(.ant-input-affix-wrapper-disabled):not( + .ant-input-affix-wrapper-borderless + ).ant-input-affix-wrapper { background-color: transparent; } `; diff --git a/datahub-web-react/src/app/auth/login.module.css b/datahub-web-react/src/app/auth/login.module.css index 37cc067c9dd20..81b933062a1a7 100644 --- a/datahub-web-react/src/app/auth/login.module.css +++ b/datahub-web-react/src/app/auth/login.module.css @@ -8,7 +8,7 @@ position: absolute; top: 40%; left: 50%; - transform: translate(-50%,-50%); + transform: translate(-50%, -50%); } .login_logo_box { @@ -28,7 +28,7 @@ .login_form_box { width: 100%; - background-color: #1C1C1C; + background-color: #1c1c1c; border: 1px solid #555555; border-radius: 5px; padding: 2em; @@ -36,8 +36,8 @@ .login_button { color: #171717; - background-color: #EEEEEE; - border: 1px solid #555555; + background-color: #eeeeee; + border: 1px solid #555555; height: 40px; font-size: 14px; } @@ -45,11 +45,11 @@ .login_button:hover { color: white; background-color: transparent; - border: 1px solid #555555; + border: 1px solid #555555; } .sso_button { - color: #EEEEEE; + color: #eeeeee; background-color: #171717; border: 1px solid #555555; height: 40px; @@ -60,4 +60,4 @@ color: black; background-color: white; border: 1px solid #555555; -} \ No newline at end of file +} diff --git a/datahub-web-react/src/app/buildEntityRegistry.ts b/datahub-web-react/src/app/buildEntityRegistry.ts index ed20722083032..0b70986672be5 100644 --- a/datahub-web-react/src/app/buildEntityRegistry.ts +++ b/datahub-web-react/src/app/buildEntityRegistry.ts @@ -19,10 +19,10 @@ import GlossaryNodeEntity from './entity/glossaryNode/GlossaryNodeEntity'; import { DataPlatformEntity } from './entity/dataPlatform/DataPlatformEntity'; import { DataProductEntity } from './entity/dataProduct/DataProductEntity'; import { DataPlatformInstanceEntity } from './entity/dataPlatformInstance/DataPlatformInstanceEntity'; -import { ERModelRelationshipEntity } from './entity/ermodelrelationships/ERModelRelationshipEntity' +import { ERModelRelationshipEntity } from './entity/ermodelrelationships/ERModelRelationshipEntity'; import { RoleEntity } from './entity/Access/RoleEntity'; import { RestrictedEntity } from './entity/restricted/RestrictedEntity'; -import {BusinessAttributeEntity} from "./entity/businessAttribute/BusinessAttributeEntity"; +import { BusinessAttributeEntity } from './entity/businessAttribute/BusinessAttributeEntity'; import { SchemaFieldPropertiesEntity } from './entity/schemaField/SchemaFieldPropertiesEntity'; export default function buildEntityRegistry() { @@ -48,9 +48,9 @@ export default function buildEntityRegistry() { registry.register(new DataPlatformEntity()); registry.register(new DataProductEntity()); registry.register(new DataPlatformInstanceEntity()); - registry.register(new ERModelRelationshipEntity()) + registry.register(new ERModelRelationshipEntity()); registry.register(new RestrictedEntity()); registry.register(new BusinessAttributeEntity()); registry.register(new SchemaFieldPropertiesEntity()); return registry; -} \ No newline at end of file +} diff --git a/datahub-web-react/src/app/businessAttribute/CreateBusinessAttributeModal.tsx b/datahub-web-react/src/app/businessAttribute/CreateBusinessAttributeModal.tsx index 61595045646c4..1ee0ca030748e 100644 --- a/datahub-web-react/src/app/businessAttribute/CreateBusinessAttributeModal.tsx +++ b/datahub-web-react/src/app/businessAttribute/CreateBusinessAttributeModal.tsx @@ -223,9 +223,9 @@ export default function CreateBusinessAttributeModal({ visible, onClose, onCreat > By default, a random UUID will be generated to uniquely identify this entity. If - you'd like to provide a custom id, you may provide it here. Note that it should be - unique across the entire Business Attributes. Be careful, you cannot easily change the id after - creation. + you'd like to provide a custom id, you may provide it here. Note that it should + be unique across the entire Business Attributes. Be careful, you cannot easily + change the id after creation. { it('renders', () => { diff --git a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx index 2a54a4a96c639..913d502972fe1 100644 --- a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx +++ b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx @@ -28,6 +28,7 @@ import { LOOKER_URN } from '../../ingest/source/builder/constants'; import { MatchedFieldList } from '../../search/matches/MatchedFieldList'; import { matchedInputFieldRenderer } from '../../search/matches/matchedInputFieldRenderer'; import { IncidentTab } from '../shared/tabs/Incident/IncidentTab'; +import { ChartQueryTab } from './ChartQueryTab'; /** * Definition of the DataHub Chart entity. @@ -110,6 +111,14 @@ export class ChartEntity implements Entity { component: ChartStatsSummarySubHeader, }} tabs={[ + { + name: 'Query', + component: ChartQueryTab, + display: { + visible: (_, chart: GetChartQuery) => (chart?.chart?.query?.rawQuery && true) || false, + enabled: (_, chart: GetChartQuery) => (chart?.chart?.query?.rawQuery && true) || false, + }, + }, { name: 'Documentation', component: DocumentationTab, diff --git a/datahub-web-react/src/app/entity/chart/ChartQueryTab.tsx b/datahub-web-react/src/app/entity/chart/ChartQueryTab.tsx new file mode 100644 index 0000000000000..7c28f4be88d8d --- /dev/null +++ b/datahub-web-react/src/app/entity/chart/ChartQueryTab.tsx @@ -0,0 +1,61 @@ +import { Typography } from 'antd'; +import React from 'react'; +import styled from 'styled-components'; +import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter'; +import { GetChartQuery } from '../../../graphql/chart.generated'; +import { ANTD_GRAY } from '../shared/constants'; +import { useBaseEntity } from '../shared/EntityContext'; +import { InfoItem } from '../shared/components/styled/InfoItem'; + +const InfoSection = styled.div` + border-bottom: 1px solid ${ANTD_GRAY[4.5]}; + padding: 16px 20px; +`; + +const InfoItemContainer = styled.div<{ justifyContent }>` + display: flex; + position: relative; + justify-content: ${(props) => props.justifyContent}; + padding: 12px 2px; +`; + +const InfoItemContent = styled.div` + padding-top: 8px; +`; + +const QueryText = styled(Typography.Paragraph)` + margin-top: 20px; + background-color: ${ANTD_GRAY[2]}; +`; + +// NOTE: Yes, using `!important` is a shame. However, the SyntaxHighlighter is applying styles directly +// to the component, so there's no way around this +const NestedSyntax = styled(SyntaxHighlighter)` + background-color: transparent !important; + border: none !important; +`; + +export function ChartQueryTab() { + const baseEntity = useBaseEntity(); + const query = baseEntity?.chart?.query?.rawQuery || 'UNKNOWN'; + const type = baseEntity?.chart?.query?.type || 'UNKNOWN'; + + return ( + <> + + Details + + + {type.toUpperCase()} + + + + + Query + + {query} + + + + ); +} diff --git a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx index 0caafb5523a20..6074bcc2f2f40 100644 --- a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx +++ b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx @@ -119,11 +119,11 @@ export class DatasetEntity implements Entity { component: ViewDefinitionTab, display: { visible: (_, dataset: GetDatasetQuery) => - dataset?.dataset?.subTypes?.typeNames + !!dataset?.dataset?.viewProperties?.logic || + !!dataset?.dataset?.subTypes?.typeNames ?.map((t) => t.toLocaleLowerCase()) - .includes(SUBTYPES.VIEW.toLocaleLowerCase()) || false, - enabled: (_, dataset: GetDatasetQuery) => - (dataset?.dataset?.viewProperties?.logic && true) || false, + .includes(SUBTYPES.VIEW.toLocaleLowerCase()), + enabled: (_, dataset: GetDatasetQuery) => !!dataset?.dataset?.viewProperties?.logic, }, }, { @@ -178,8 +178,7 @@ export class DatasetEntity implements Entity { }, }, { - name: 'Runs', - // TODO: Rename this to DatasetRunsTab. + name: 'Runs', // TODO: Rename this to DatasetRunsTab. component: OperationsTab, display: { visible: (_, dataset: GetDatasetQuery) => { @@ -234,7 +233,7 @@ export class DatasetEntity implements Entity { { component: SidebarViewDefinitionSection, display: { - visible: (_, dataset: GetDatasetQuery) => (dataset?.dataset?.viewProperties?.logic && true) || false, + visible: (_, dataset: GetDatasetQuery) => !!dataset?.dataset?.viewProperties?.logic, }, }, { @@ -249,8 +248,7 @@ export class DatasetEntity implements Entity { }, { component: DataProductSection, - }, - // TODO: Add back once entity-level recommendations are complete. + }, // TODO: Add back once entity-level recommendations are complete. // { // component: SidebarRecommendationsSection, // }, diff --git a/datahub-web-react/src/app/entity/domain/DataProductsTab/CreateDataProductModal.tsx b/datahub-web-react/src/app/entity/domain/DataProductsTab/CreateDataProductModal.tsx index 0610fbfa7a770..a2347928136e5 100644 --- a/datahub-web-react/src/app/entity/domain/DataProductsTab/CreateDataProductModal.tsx +++ b/datahub-web-react/src/app/entity/domain/DataProductsTab/CreateDataProductModal.tsx @@ -50,7 +50,7 @@ export default function CreateDataProductModal({ domain, onCreateDataProduct, on onClose(); } }) - .catch(( error ) => { + .catch((error) => { onClose(); message.destroy(); message.error({ content: `Failed to create Data Product: ${error.message}.` }); diff --git a/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductAdvancedOption.tsx b/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductAdvancedOption.tsx index a077a0308af1f..c3952fbaf5cb0 100644 --- a/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductAdvancedOption.tsx +++ b/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductAdvancedOption.tsx @@ -1,9 +1,8 @@ -import React from "react"; -import { Collapse, Form, Input, Typography } from "antd"; -import styled from "styled-components"; +import React from 'react'; +import { Collapse, Form, Input, Typography } from 'antd'; +import styled from 'styled-components'; import { validateCustomUrnId } from '../../../shared/textUtil'; -import { DataProductBuilderFormProps } from "./types"; - +import { DataProductBuilderFormProps } from './types'; const FormItem = styled(Form.Item)` .ant-form-item-label { @@ -23,8 +22,7 @@ const AdvancedLabel = styled(Typography.Text)` color: #373d44; `; -export function DataProductAdvancedOption({builderState, updateBuilderState }: DataProductBuilderFormProps){ - +export function DataProductAdvancedOption({ builderState, updateBuilderState }: DataProductBuilderFormProps) { function updateDataProductId(id: string) { updateBuilderState({ ...builderState, @@ -54,9 +52,9 @@ export function DataProductAdvancedOption({builderState, updateBuilderState }: D }), ]} > - updateDataProductId(e.target.value)} /> @@ -64,5 +62,5 @@ export function DataProductAdvancedOption({builderState, updateBuilderState }: D - ) -} \ No newline at end of file + ); +} diff --git a/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductBuilderForm.tsx b/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductBuilderForm.tsx index 98bb09098a36e..695267080c92f 100644 --- a/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductBuilderForm.tsx +++ b/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductBuilderForm.tsx @@ -43,7 +43,7 @@ export default function DataProductBuilderForm({ builderState, updateBuilderStat Description}> - + ); } diff --git a/datahub-web-react/src/app/entity/domain/DataProductsTab/types.ts b/datahub-web-react/src/app/entity/domain/DataProductsTab/types.ts index fe22e3ed9a2a4..2015b97f1433b 100644 --- a/datahub-web-react/src/app/entity/domain/DataProductsTab/types.ts +++ b/datahub-web-react/src/app/entity/domain/DataProductsTab/types.ts @@ -7,4 +7,4 @@ export type DataProductBuilderState = { export type DataProductBuilderFormProps = { builderState: DataProductBuilderState; updateBuilderState: (newState: DataProductBuilderState) => void; -}; \ No newline at end of file +}; diff --git a/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx b/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx index 91005f17b80c7..aece3db1312af 100644 --- a/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx +++ b/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx @@ -7,7 +7,10 @@ import { GenericEntityProperties } from '../shared/types'; import { ERModelRelationshipPreviewCard } from './preview/ERModelRelationshipPreviewCard'; import ermodelrelationshipIcon from '../../../images/ermodelrelationshipIcon.svg'; import { ERModelRelationshipTab } from '../shared/tabs/ERModelRelationship/ERModelRelationshipTab'; -import { useGetErModelRelationshipQuery, useUpdateErModelRelationshipMutation } from '../../../graphql/ermodelrelationship.generated'; +import { + useGetErModelRelationshipQuery, + useUpdateErModelRelationshipMutation, +} from '../../../graphql/ermodelrelationship.generated'; import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab'; import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab'; import { SidebarAboutSection } from '../shared/containers/profile/sidebar/AboutSection/SidebarAboutSection'; @@ -108,7 +111,9 @@ export class ERModelRelationshipEntity implements Entity { {data.properties?.name || data.editableProperties?.name || ''} + + {data.properties?.name || data.editableProperties?.name || ''} + } description={data?.editableProperties?.description || ''} owners={data.ownership?.owners} diff --git a/datahub-web-react/src/app/entity/ermodelrelationships/preview/ERModelRelationshipAction.less b/datahub-web-react/src/app/entity/ermodelrelationships/preview/ERModelRelationshipAction.less index 7ac539d7a6a1e..0f63ee197fecb 100644 --- a/datahub-web-react/src/app/entity/ermodelrelationships/preview/ERModelRelationshipAction.less +++ b/datahub-web-react/src/app/entity/ermodelrelationships/preview/ERModelRelationshipAction.less @@ -1,4 +1,4 @@ -@import "../../../../../node_modules/antd/dist/antd.less"; +@import '../../../../../node_modules/antd/dist/antd.less'; .joinName { width: 385px; @@ -9,4 +9,4 @@ line-height: 24px; align-items: center; color: #262626; -} \ No newline at end of file +} diff --git a/datahub-web-react/src/app/entity/ermodelrelationships/preview/ERModelRelationshipPreviewCard.tsx b/datahub-web-react/src/app/entity/ermodelrelationships/preview/ERModelRelationshipPreviewCard.tsx index 33669485f18c6..715f935685d54 100644 --- a/datahub-web-react/src/app/entity/ermodelrelationships/preview/ERModelRelationshipPreviewCard.tsx +++ b/datahub-web-react/src/app/entity/ermodelrelationships/preview/ERModelRelationshipPreviewCard.tsx @@ -32,7 +32,9 @@ export const ERModelRelationshipPreviewCard = ({ name={name || ''} urn={urn} description={description || ''} - logoComponent={ERModelRelationship} + logoComponent={ + ERModelRelationship + } tags={globalTags || undefined} glossaryTerms={glossaryTerms || undefined} owners={owners} diff --git a/datahub-web-react/src/app/entity/group/EditGroupDescriptionModal.tsx b/datahub-web-react/src/app/entity/group/EditGroupDescriptionModal.tsx index a898a73c254ef..356daf584d9f7 100644 --- a/datahub-web-react/src/app/entity/group/EditGroupDescriptionModal.tsx +++ b/datahub-web-react/src/app/entity/group/EditGroupDescriptionModal.tsx @@ -22,19 +22,18 @@ export default function EditGroupDescriptionModal({ stagedDescription, }: Props) { const [form] = Form.useForm(); - const [aboutText,setAboutText] = useState(stagedDescription) + const [aboutText, setAboutText] = useState(stagedDescription); function updateDescription(description: string) { - setAboutText(aboutText) + setAboutText(aboutText); setStagedDescription(description); - } const saveDescription = () => { onSaveAboutMe(); onClose(); }; - + return ( { + const { updateTitle } = useBrowserTitle(); + + useEffect(() => { // You can use the title and updateTitle function here // For example, updating the title when the component mounts - if(name){ + if (name) { updateTitle(`Group | ${name}`); } // // Don't forget to clean up the title when the component unmounts return () => { - if(name){ // added to condition for rerendering issue + if (name) { + // added to condition for rerendering issue updateTitle(''); } }; @@ -216,7 +217,7 @@ export default function GroupInfoSidebar({ sideBarData, refetch }: Props) { urn, email, slack, - photoUrl + photoUrl, }; // About Text save diff --git a/datahub-web-react/src/app/entity/ownership/table/ActionsColumn.tsx b/datahub-web-react/src/app/entity/ownership/table/ActionsColumn.tsx index e08853ad150bf..cf4bf9a0fddf4 100644 --- a/datahub-web-react/src/app/entity/ownership/table/ActionsColumn.tsx +++ b/datahub-web-react/src/app/entity/ownership/table/ActionsColumn.tsx @@ -48,9 +48,9 @@ export const ActionsColumn = ({ ownershipType, setIsOpen, setOwnershipType, refe setOwnershipType(ownershipType); }; - const onCopy=() => { + const onCopy = () => { navigator.clipboard.writeText(ownershipType.urn); - } + }; const [deleteOwnershipTypeMutation] = useDeleteOwnershipTypeMutation(); @@ -125,8 +125,7 @@ export const ActionsColumn = ({ ownershipType, setIsOpen, setOwnershipType, refe const key = e.key as string; if (key === 'edit') { editOnClick(); - } - else if( key === 'copy') { + } else if (key === 'copy') { onCopy(); } }; diff --git a/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx b/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx index 91638d4997003..7e74b43e68afb 100644 --- a/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx +++ b/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx @@ -34,11 +34,7 @@ export class SchemaFieldPropertiesEntity implements Entity { renderProfile = (_: string) => <>; renderPreview = (previewType: PreviewType, data: SchemaFieldEntity) => ( - + ); renderSearch = (result: SearchResult) => this.renderPreview(PreviewType.SEARCH, result.entity as SchemaFieldEntity); diff --git a/datahub-web-react/src/app/entity/schemaField/preview/Preview.tsx b/datahub-web-react/src/app/entity/schemaField/preview/Preview.tsx index 3f24b3a06e3a4..b22e988c76672 100644 --- a/datahub-web-react/src/app/entity/schemaField/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/schemaField/preview/Preview.tsx @@ -20,7 +20,9 @@ export const Preview = ({ }): JSX.Element => { const entityRegistry = useEntityRegistry(); - const url = `${entityRegistry.getEntityUrl(EntityType.Dataset, datasetUrn)}/${encodeURIComponent('Schema')}?schemaFilter=${encodeURIComponent(name)}`; + const url = `${entityRegistry.getEntityUrl(EntityType.Dataset, datasetUrn)}/${encodeURIComponent( + 'Schema', + )}?schemaFilter=${encodeURIComponent(name)}`; return ( ); -}; \ No newline at end of file +}; diff --git a/datahub-web-react/src/app/entity/shared/EntityContext.ts b/datahub-web-react/src/app/entity/shared/EntityContext.ts index c564d73c7441f..abc7fcfa6cced 100644 --- a/datahub-web-react/src/app/entity/shared/EntityContext.ts +++ b/datahub-web-react/src/app/entity/shared/EntityContext.ts @@ -22,17 +22,17 @@ export function useEntityContext() { return useContext(EntityContext); } -export const useBaseEntity = (): T => { +export const useBaseEntity = (): T => { const { baseEntity } = useContext(EntityContext); return baseEntity as T; }; -export const useDataNotCombinedWithSiblings = (): T => { +export const useDataNotCombinedWithSiblings = (): T => { const { dataNotCombinedWithSiblings } = useContext(EntityContext); return dataNotCombinedWithSiblings as T; }; -export const useEntityUpdate = (): UpdateEntityType | null | undefined => { +export const useEntityUpdate = (): UpdateEntityType | null | undefined => { const { updateEntity } = useContext(EntityContext); return updateEntity; }; diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/UpdateDeprecationModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/UpdateDeprecationModal.tsx index 25527497b33a8..01287c2b367bf 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/UpdateDeprecationModal.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/UpdateDeprecationModal.tsx @@ -71,11 +71,11 @@ export const UpdateDeprecationModal = ({ urns, onClose, refetch }: Props) => { } - width='40%' + width="40%" >
- + diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/useDeleteEntity.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/useDeleteEntity.tsx index 171a36b1cfbcc..aa00e8ebc879d 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/useDeleteEntity.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/useDeleteEntity.tsx @@ -29,16 +29,10 @@ function useDeleteEntity( const { isInGlossaryContext, urnsToUpdate, setUrnsToUpdate } = useGlossaryEntityData(); const { handleDeleteDomain } = useHandleDeleteDomain({ entityData, urn }); - const maybeDeleteEntity = getDeleteEntityMutation(type)(); - const deleteEntity = (maybeDeleteEntity && maybeDeleteEntity[0]) || undefined; - const client = maybeDeleteEntity?.[1].client; + const [deleteEntity, { client }] = getDeleteEntityMutation(type)() ?? [undefined, { client: undefined }]; function handleDeleteEntity() { - deleteEntity?.({ - variables: { - urn, - }, - }) + deleteEntity?.({ variables: { urn } }) .then(() => { analytics.event({ type: EventType.DeleteEntityEvent, @@ -56,10 +50,6 @@ function useDeleteEntity( handleDeleteDomain(); } - if (client && entityData.type === EntityType.GlossaryTerm && entityData?.parentNodes?.nodes) { - removeTermFromGlossaryNode(client, entityData.parentNodes.nodes[0].urn, urn); - } - setTimeout( () => { setHasBeenDeleted(true); @@ -67,6 +57,9 @@ function useDeleteEntity( if (isInGlossaryContext) { const parentNodeToUpdate = getParentNodeToUpdate(entityData, type); updateGlossarySidebar([parentNodeToUpdate], urnsToUpdate, setUrnsToUpdate); + if (client) { + removeTermFromGlossaryNode(client, parentNodeToUpdate, urn); + } } if (!hideMessage) { message.success({ diff --git a/datahub-web-react/src/app/entity/shared/ExternalUrlButton.tsx b/datahub-web-react/src/app/entity/shared/ExternalUrlButton.tsx index dce74c02cdb34..d821cbfc01355 100644 --- a/datahub-web-react/src/app/entity/shared/ExternalUrlButton.tsx +++ b/datahub-web-react/src/app/entity/shared/ExternalUrlButton.tsx @@ -5,6 +5,8 @@ import UrlButton from './UrlButton'; const GITHUB_LINK = 'github.com'; const GITHUB = 'GitHub'; +const GITLAB_LINK = 'gitlab.com'; +const GITLAB = 'GitLab'; interface Props { externalUrl: string; @@ -26,6 +28,8 @@ export default function ExternalUrlButton({ externalUrl, platformName, entityTyp let displayedName = platformName; if (externalUrl.toLocaleLowerCase().includes(GITHUB_LINK)) { displayedName = GITHUB; + } else if (externalUrl.toLocaleLowerCase().includes(GITLAB_LINK)) { + displayedName = GITLAB; } return ( diff --git a/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx b/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx index 9ec2aab193aa0..08e9636f760de 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx @@ -165,8 +165,7 @@ export const DeprecationPill = ({ deprecation, urn, refetch, showUndeprecate }: {expanded || !overLimit ? ( <> - { - deprecation?.note && deprecation?.note !== '' && + {deprecation?.note && deprecation?.note !== '' && ( <> @@ -181,7 +180,7 @@ export const DeprecationPill = ({ deprecation, urn, refetch, showUndeprecate }: )} - } + )} ) : ( <> diff --git a/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/CreateERModelRelationModal.less b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/CreateERModelRelationModal.less index b50d3debaf1ef..8c1f29aa7fc77 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/CreateERModelRelationModal.less +++ b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/CreateERModelRelationModal.less @@ -1,12 +1,12 @@ -@import "../../../../../../../node_modules/antd/dist/antd.less"; +@import '../../../../../../../node_modules/antd/dist/antd.less'; .CreateERModelRelationModal { .ermodelrelation-name { padding: 8px 16px; width: 948.5px !important; height: 40px !important; - background: #FFFFFF; - border: 1px solid #D9D9D9; + background: #ffffff; + border: 1px solid #d9d9d9; box-shadow: 0px 2px 0px rgba(0, 0, 0, 0.016); border-radius: 2px; align-items: center; @@ -18,8 +18,8 @@ max-width: 370px; min-width: 370px; height: 38px; - background: #FFFFFF; - border: 1px solid #D9D9D9; + background: #ffffff; + border: 1px solid #d9d9d9; box-shadow: 0px 2px 0px rgba(0, 0, 0, 0.016); border-radius: 2px; } @@ -27,8 +27,8 @@ box-sizing: border-box; width: 1000px; height: 765px; - background: #FFFFFF; - border: 1px solid #ADC0D7; + background: #ffffff; + border: 1px solid #adc0d7; box-shadow: 0px 0px 10px rgba(0, 0, 0, 0.15); border-radius: 8px; left: -215px; @@ -58,7 +58,7 @@ color: #000000; padding-top: 4px; } - .all-content-heading{ + .all-content-heading { width: 380px; height: 16px; margin-top: 16px; @@ -68,10 +68,10 @@ font-weight: 700; font-size: 14px; line-height: 16px; - color: #1B2F41; + color: #1b2f41; flex: none; } - .all-table-heading{ + .all-table-heading { width: 380px; height: 16px; margin-bottom: 8px; @@ -80,11 +80,11 @@ font-weight: 700; font-size: 14px; line-height: 16px; - color: #1B2F41; + color: #1b2f41; flex: none; } - - .field-heading{ + + .field-heading { height: 16px; margin-top: 32px; margin-bottom: 8px; @@ -93,16 +93,16 @@ font-weight: 700; font-size: 14px; line-height: 16px; - color: #1B2F41; + color: #1b2f41; } - .all-information{ + .all-information { width: 680px; height: 24px; font-family: 'Arial'; font-style: normal; font-weight: 400; font-size: 16px; - color: #1B2F41; + color: #1b2f41; } .techNameDisplay { font-size: 14px; @@ -127,7 +127,7 @@ padding-right: 25px; border-top: 0px; } - + .ant-btn-link { padding-left: 0px !important; padding-right: 1px !important; @@ -135,7 +135,7 @@ font-style: normal !important; font-weight: 400 !important; font-size: 14px !important; - color: #1890FF !important; + color: #1890ff !important; } .add-btn-link { padding-left: 865px !important; @@ -146,7 +146,7 @@ font-style: normal !important; font-weight: 700 !important; font-size: 12px !important; - color: #1890FF !important; + color: #1890ff !important; line-height: 20px; } @@ -155,20 +155,21 @@ margin-left: 440px; width: 85px; height: 40px !important; - background: #FFFFFF; - border: 1px solid #D9D9D9 !important; + background: #ffffff; + border: 1px solid #d9d9d9 !important; border-radius: 5px; color: #262626; } - - .submit-btn, .submit-btn:hover { + + .submit-btn, + .submit-btn:hover { margin-left: 28px; //margin-top: 6px; width: 86px; height: 40px; - background: #1890FF; + background: #1890ff; border: none; - color: #FFFFFF; + color: #ffffff; } .footer-parent-div { padding-left: 8px; @@ -183,11 +184,11 @@ min-width: 373px !important; font-size: 14px; line-height: 22px; - font-family: 'Roboto Mono',monospace; + font-family: 'Roboto Mono', monospace; font-weight: 400; background: white; font-style: normal; - color: #000000D9; + color: #000000d9; } .ermodelrelation-details-ta { height: 95px; @@ -213,13 +214,13 @@ font-weight: 500; font-size: 14px; line-height: 22px; - color: #1B2F41; + color: #1b2f41; align-items: center; padding: 16px; gap: 4px; isolation: isolate; height: 56px !important; - background: #FFFFFF; + background: #ffffff; border-color: rgba(0, 0, 0, 0.12); } .ant-table-tbody > tr td { @@ -233,11 +234,12 @@ color: rgba(0, 0, 0, 0.85); border-color: rgba(0, 0, 0, 0.12); } - td:nth-child(1), td:nth-child(3){ + td:nth-child(1), + td:nth-child(3) { max-width: 400px !important; min-width: 400px !important; } - .titleNameDisplay{ + .titleNameDisplay { max-width: 360px; overflow: hidden; white-space: nowrap; @@ -247,11 +249,11 @@ font-size: 14px; padding: 4px 0; } - .firstRow{ + .firstRow { display: flex; justify-content: left; } - + .editableNameDisplay { display: block; overflow-wrap: break-word; @@ -267,11 +269,13 @@ line-height: 16px; color: #595959; } - td:nth-child(2), th:nth-child(2){ + td:nth-child(2), + th:nth-child(2) { min-width: 44px !important; max-width: 44px !important; } - td:nth-child(4), th:nth-child(4){ + td:nth-child(4), + th:nth-child(4) { min-width: 75px !important; max-width: 75px !important; } @@ -280,15 +284,15 @@ border-collapse: collapse; } .SelectedRow { - background-color: #ECF2F8; + background-color: #ecf2f8; } } } .cancel-modal { .ant-btn-primary { - color: #FFFFFF; - background: #1890FF; + color: #ffffff; + background: #1890ff; border: none; box-shadow: none; } -} \ No newline at end of file +} diff --git a/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/CreateERModelRelationModal.tsx b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/CreateERModelRelationModal.tsx index a6f84b8c8fc5c..dd6cbc3698705 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/CreateERModelRelationModal.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/CreateERModelRelationModal.tsx @@ -5,11 +5,19 @@ import { PlusOutlined } from '@ant-design/icons'; import arrow from '../../../../../../images/Arrow.svg'; import './CreateERModelRelationModal.less'; import { EntityType, ErModelRelationship, OwnerEntityType } from '../../../../../../types.generated'; -import { useCreateErModelRelationshipMutation, useUpdateErModelRelationshipMutation } from '../../../../../../graphql/ermodelrelationship.generated'; +import { + useCreateErModelRelationshipMutation, + useUpdateErModelRelationshipMutation, +} from '../../../../../../graphql/ermodelrelationship.generated'; import { useUserContext } from '../../../../../context/useUserContext'; import { EditableRow } from './EditableRow'; import { EditableCell } from './EditableCell'; -import { checkDuplicateERModelRelation, getDatasetName, ERModelRelationDataType, validateERModelRelation } from './ERModelRelationUtils'; +import { + checkDuplicateERModelRelation, + getDatasetName, + ERModelRelationDataType, + validateERModelRelation, +} from './ERModelRelationUtils'; import { useGetSearchResultsQuery } from '../../../../../../graphql/search.generated'; import { useAddOwnerMutation } from '../../../../../../graphql/mutations.generated'; @@ -52,7 +60,10 @@ export const CreateERModelRelationModal = ({ const [details, setDetails] = useState(editERModelRelation?.editableProperties?.description || ''); const [ermodelrelationName, setERModelRelationName] = useState( - editERModelRelation?.editableProperties?.name || editERModelRelation?.properties?.name || editERModelRelation?.id || '', + editERModelRelation?.editableProperties?.name || + editERModelRelation?.properties?.name || + editERModelRelation?.id || + '', ); const [tableData, setTableData] = useState( editERModelRelation?.properties?.relationshipFieldMappings?.map((item, index) => { @@ -116,11 +127,11 @@ export const CreateERModelRelationModal = ({ destination: table2Dataset?.urn || '', name: ermodelrelationName, relationshipFieldmappings: tableData.map((r) => { - return { - sourceField: r.field1Name, - destinationField: r.field2Name, - }; - }), + return { + sourceField: r.field1Name, + destinationField: r.field2Name, + }; + }), created: true, }, editableProperties: { @@ -171,12 +182,12 @@ export const CreateERModelRelationModal = ({ createdBy: editERModelRelation?.properties?.createdActor?.urn || user?.urn, createdAt: editERModelRelation?.properties?.createdTime || 0, relationshipFieldmappings: tableData.map((r) => { - return { - sourceField: r.field1Name, - destinationField: r.field2Name, - }; - }), - }, + return { + sourceField: r.field1Name, + destinationField: r.field2Name, + }; + }), + }, editableProperties: { name: ermodelrelationName, description: details, @@ -203,7 +214,12 @@ export const CreateERModelRelationModal = ({ }); }; const onSubmit = async () => { - const errors = validateERModelRelation(ermodelrelationName, tableData, isEditing, getSearchResultsERModelRelations); + const errors = validateERModelRelation( + ermodelrelationName, + tableData, + isEditing, + getSearchResultsERModelRelations, + ); if ((await errors).length > 0) { const err = (await errors).join(`, `); message.error(err); @@ -368,19 +384,25 @@ export const CreateERModelRelationModal = ({ }, { validator: (_, value) => - checkDuplicateERModelRelation(getSearchResultsERModelRelations, value?.trim()).then((result) => { - return result === true && !isEditing - ? Promise.reject( - new Error( - 'This ER-Model-Relationship name already exists. A unique name for each ER-Model-Relationship is required.', - ), - ) - : Promise.resolve(); - }), + checkDuplicateERModelRelation(getSearchResultsERModelRelations, value?.trim()).then( + (result) => { + return result === true && !isEditing + ? Promise.reject( + new Error( + 'This ER-Model-Relationship name already exists. A unique name for each ER-Model-Relationship is required.', + ), + ) + : Promise.resolve(); + }, + ), }, ]} > - setERModelRelationName(e.target.value)} /> + setERModelRelationName(e.target.value)} + />

Fields

tr th { padding: 16px; gap: 4px; - background: #FFFFFF; + background: #ffffff; border-width: 1px 0px 1px 1px; border-style: solid; border-color: rgba(0, 0, 0, 0.12); @@ -133,17 +133,17 @@ .ant-table-tbody > tr td { font-size: 14px; line-height: 22px; - font-family: 'Roboto Mono',monospace; + font-family: 'Roboto Mono', monospace; font-weight: 400; background: white; font-style: normal; - color: #000000D9; + color: #000000d9; padding: 16px; border-width: 0px 0px 1px 1px; border-style: solid; border-color: rgba(0, 0, 0, 0.12); } - .firstRow{ + .firstRow { display: flex; justify-content: left; } @@ -152,7 +152,7 @@ font-style: normal; font-weight: 600; line-height: 22px; - color: #1B2F41; + color: #1b2f41; } .editableNameDisplay { color: #595959; @@ -163,14 +163,13 @@ line-height: normal; } .SelectedRow { - background-color: #ECF2F8; + background-color: #ecf2f8; } } } -.radioButton{ - .ant-radio-inner{ +.radioButton { + .ant-radio-inner { margin-top: 20px; border-color: #4a5568; } } - diff --git a/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/ERModelRelationPreview.tsx b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/ERModelRelationPreview.tsx index a033f9d4a3574..b360f03bb5b28 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/ERModelRelationPreview.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/ERModelRelationPreview.tsx @@ -51,7 +51,8 @@ export const ERModelRelationPreview = ({ ermodelrelationData, baseEntityUrn, pre shuffleFlag && prePageType !== 'ERModelRelationship' ? ermodelrelationData?.properties?.source?.urn : ermodelrelationData?.properties?.destination?.urn; - const ermodelrelationHeader = ermodelrelationData?.editableProperties?.name || ermodelrelationData?.properties?.name || ''; + const ermodelrelationHeader = + ermodelrelationData?.editableProperties?.name || ermodelrelationData?.properties?.name || ''; function getFieldMap(): ERModelRelationRecord[] { const newData = [] as ERModelRelationRecord[]; if (shuffleFlag && prePageType !== 'ERModelRelationship') { @@ -147,7 +148,12 @@ export const ERModelRelationPreview = ({ ermodelrelationData, baseEntityUrn, pre

{ermodelrelationHeader}

{prePageType === 'Dataset' && ( - - } trigger={['click']}> - - - + <> + {showMenu && ( + + + + {(record.platform.properties?.logoUrl && ( + + )) || ( + + {record.platform.properties?.displayName || + capitalizeFirstLetterOnly(record.platform.name)} + + )} + + + + } trigger={['click']}> + + + + )} + ), }, ]; @@ -168,18 +263,36 @@ export const DatasetAssertionsList = ({ assertions, onDelete }: Props) => { locale={{ emptyText: , }} - expandable={{ - defaultExpandAllRows: false, - expandRowByClick: true, - expandedRowRender: (record) => { - return ; - }, - expandIcon: ({ expanded, onExpand, record }: any) => - expanded ? ( - onExpand(record, e)} /> - ) : ( - onExpand(record, e)} /> - ), + expandable={ + showSelect + ? {} + : { + defaultExpandAllRows: false, + expandRowByClick: true, + expandedRowRender: (record) => { + return ( + + ); + }, + expandIcon: ({ expanded, onExpand, record }: any) => + expanded ? ( + onExpand(record, e)} /> + ) : ( + onExpand(record, e)} /> + ), + } + } + onRow={(record) => { + return { + onClick: (_) => { + if (showSelect) { + onSelect?.(record.urn as string); + } + }, + }; }} showHeader={false} pagination={false} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/FieldAssertionDescription.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/FieldAssertionDescription.tsx new file mode 100644 index 0000000000000..a104903dc7bc2 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/FieldAssertionDescription.tsx @@ -0,0 +1,31 @@ +import React from 'react'; +import { Typography } from 'antd'; +import { FieldAssertionInfo } from '../../../../../../types.generated'; +import { + getFieldDescription, + getFieldOperatorDescription, + getFieldParametersDescription, + getFieldTransformDescription, +} from './fieldDescriptionUtils'; + +type Props = { + assertionInfo: FieldAssertionInfo; +}; + +/** + * A human-readable description of a Field Assertion. + */ +export const FieldAssertionDescription = ({ assertionInfo }: Props) => { + const field = getFieldDescription(assertionInfo); + const operator = getFieldOperatorDescription(assertionInfo); + const transform = getFieldTransformDescription(assertionInfo); + const parameters = getFieldParametersDescription(assertionInfo); + + return ( + + {transform} + {transform ? ' of ' : ''} + {field} {operator} {parameters} + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/SqlAssertionDescription.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/SqlAssertionDescription.tsx new file mode 100644 index 0000000000000..047f7c7db28f6 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/SqlAssertionDescription.tsx @@ -0,0 +1,16 @@ +import React from 'react'; +import { Typography } from 'antd'; +import { AssertionInfo } from '../../../../../../types.generated'; + +type Props = { + assertionInfo: AssertionInfo; +}; + +/** + * A human-readable description of a SQL Assertion. + */ +export const SqlAssertionDescription = ({ assertionInfo }: Props) => { + const { description } = assertionInfo; + + return {description}; +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx index b4f77196edbb1..92af9bfc2b567 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx @@ -2,13 +2,15 @@ import React, { useEffect } from 'react'; import { Button } from 'antd'; import { useHistory, useLocation } from 'react-router'; import styled from 'styled-components'; -import { FileDoneOutlined, FileProtectOutlined } from '@ant-design/icons'; +import { AuditOutlined, FileDoneOutlined, FileProtectOutlined } from '@ant-design/icons'; import { useEntityData } from '../../../EntityContext'; import { TestResults } from './TestResults'; import { Assertions } from './Assertions'; import TabToolbar from '../../../components/styled/TabToolbar'; import { useGetValidationsTab } from './useGetValidationsTab'; import { ANTD_GRAY } from '../../../constants'; +import { useAppConfig } from '../../../../../useAppConfig'; +import { DataContractTab } from './contract/DataContractTab'; const TabTitle = styled.span` margin-left: 4px; @@ -22,6 +24,7 @@ const TabButton = styled(Button)<{ selected: boolean }>` enum TabPaths { ASSERTIONS = 'Assertions', TESTS = 'Tests', + DATA_CONTRACT = 'Data Contract', } const DEFAULT_TAB = TabPaths.ASSERTIONS; @@ -33,6 +36,7 @@ export const ValidationsTab = () => { const { entityData } = useEntityData(); const history = useHistory(); const { pathname } = useLocation(); + const appConfig = useAppConfig(); const totalAssertions = (entityData as any)?.assertions?.total; const passingTests = (entityData as any)?.testResults?.passing || []; @@ -77,6 +81,22 @@ export const ValidationsTab = () => { }, ]; + if (appConfig.config.featureFlags?.dataContractsEnabled) { + // If contracts feature is enabled, add to list. + tabs.push({ + title: ( + <> + + + Data Contract + + ), + path: TabPaths.DATA_CONTRACT, + content: , + disabled: false, + }); + } + return ( <> diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/VolumeAssertionDescription.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/VolumeAssertionDescription.tsx new file mode 100644 index 0000000000000..26634c459df0d --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/VolumeAssertionDescription.tsx @@ -0,0 +1,41 @@ +import React from 'react'; +import { Typography } from 'antd'; +import { + IncrementingSegmentRowCountChange, + RowCountChange, + VolumeAssertionInfo, +} from '../../../../../../types.generated'; +import { + getIsRowCountChange, + getOperatorDescription, + getParameterDescription, + getValueChangeTypeDescription, + getVolumeTypeDescription, + getVolumeTypeInfo, +} from './utils'; + +type Props = { + assertionInfo: VolumeAssertionInfo; +}; + +/** + * A human-readable description of a Volume Assertion. + */ +export const VolumeAssertionDescription = ({ assertionInfo }: Props) => { + const volumeType = assertionInfo.type; + const volumeTypeInfo = getVolumeTypeInfo(assertionInfo); + const volumeTypeDescription = getVolumeTypeDescription(volumeType); + const operatorDescription = volumeTypeInfo ? getOperatorDescription(volumeTypeInfo.operator) : ''; + const parameterDescription = volumeTypeInfo ? getParameterDescription(volumeTypeInfo.parameters) : ''; + const valueChangeTypeDescription = getIsRowCountChange(volumeType) + ? getValueChangeTypeDescription((volumeTypeInfo as RowCountChange | IncrementingSegmentRowCountChange).type) + : 'rows'; + + return ( +
+ + Table {volumeTypeDescription} {operatorDescription} {parameterDescription} {valueChangeTypeDescription} + +
+ ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/assertionUtils.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/assertionUtils.tsx index 1eaacb36515a1..341742f407f73 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/assertionUtils.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/assertionUtils.tsx @@ -18,7 +18,7 @@ export const sortAssertions = (a, b) => { if (!b.runEvents?.runEvents?.length) { return -1; } - return b.runEvents.runEvents[0].timestampMillis - a.runEvents.runEvents[0].timestampMillis; + return b.runEvents?.runEvents[0]?.timestampMillis - a.runEvents?.runEvents[0]?.timestampMillis; }; /** diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractAssertionStatus.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractAssertionStatus.tsx new file mode 100644 index 0000000000000..c36bef09cdb68 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractAssertionStatus.tsx @@ -0,0 +1,44 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Tooltip } from 'antd'; +import { StopOutlined } from '@ant-design/icons'; +import { Assertion, AssertionResultType } from '../../../../../../../types.generated'; +import { + StyledCheckOutlined, + StyledClockCircleOutlined, + StyledCloseOutlined, + StyledExclamationOutlined, +} from '../shared/styledComponents'; + +const StatusContainer = styled.div` + width: 100%; + display: flex; + justify-content: center; +`; + +type Props = { + assertion: Assertion; +}; + +export const DataContractAssertionStatus = ({ assertion }: Props) => { + const latestRun = (assertion.runEvents?.runEvents?.length && assertion.runEvents?.runEvents[0]) || undefined; + const latestResultType = latestRun?.result?.type || undefined; + + return ( + + {latestResultType === undefined && } + + {latestResultType === AssertionResultType.Success && } + + + {latestResultType === AssertionResultType.Failure && } + + + {latestResultType === AssertionResultType.Error && } + + + {latestResultType === AssertionResultType.Init && } + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractEmptyState.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractEmptyState.tsx new file mode 100644 index 0000000000000..36004cb1351f3 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractEmptyState.tsx @@ -0,0 +1,86 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Button, Typography } from 'antd'; +import { PlusOutlined } from '@ant-design/icons'; + +import { ANTD_GRAY } from '../../../../constants'; + +const Container = styled.div``; + +const Summary = styled.div` + width: 100%; + padding-left: 40px; + padding-top: 20px; + padding-bottom: 20px; + display: flex; + align-items: center; + justify-content: space-between; + border-bottom: 1px solid ${ANTD_GRAY[4.5]}; + box-shadow: 0px 2px 6px 0px #0000000d; +`; + +const SummaryDescription = styled.div` + display: flex; + align-items: center; +`; + +const SummaryMessage = styled.div` + display: inline-block; + margin-left: 20px; + max-width: 350px; +`; + +const SummaryTitle = styled(Typography.Title)` + && { + padding-bottom: 0px; + margin-bottom: 4px; + } +`; + +const Actions = styled.div` + margin: 12px; + margin-right: 20px; +`; + +const CreateButton = styled(Button)` + margin-right: 12px; + border-color: ${(props) => props.theme.styles['primary-color']}; + color: ${(props) => props.theme.styles['primary-color']}; + letter-spacing: 2px; + &&:hover { + color: white; + background-color: ${(props) => props.theme.styles['primary-color']}; + border-color: ${(props) => props.theme.styles['primary-color']}; + } +`; + +type Props = { + showContractBuilder: () => void; +}; + +export const DataContractEmptyState = ({ showContractBuilder }: Props) => { + return ( + + + + + + No contract found +
+ + A contract does not yet exist for this dataset + +
+ + + + + + + CREATE + + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractSummary.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractSummary.tsx new file mode 100644 index 0000000000000..9b684486cb5ce --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractSummary.tsx @@ -0,0 +1,95 @@ +import React from 'react'; +import styled from 'styled-components'; +import { EditFilled } from '@ant-design/icons'; +import { Button, Typography } from 'antd'; +import { DataContractState } from '../../../../../../../types.generated'; +import { AssertionStatusSummary } from '../types'; +import { getContractSummaryIcon, getContractSummaryTitle, getContractSummaryMessage } from './utils'; +import { ANTD_GRAY } from '../../../../constants'; + +const SummaryHeader = styled.div` + width: 100%; + padding-left: 40px; + padding-top: 20px; + padding-bottom: 20px; + display: flex; + align-items: center; + justify-content: space-between; + border-bottom: 1px solid ${ANTD_GRAY[4.5]}; +`; + +const SummaryContainer = styled.div``; + +const SummaryDescription = styled.div` + display: flex; + align-items: center; +`; + +const SummaryMessage = styled.div` + display: inline-block; + margin-left: 20px; +`; + +const SummaryTitle = styled(Typography.Title)` + && { + padding-bottom: 0px; + margin-bottom: 0px; + } +`; + +const Actions = styled.div` + margin: 12px; + margin-right: 20px; +`; + +const CreateButton = styled(Button)` + display: flex; + align-items: center; + gap: 0.3rem; + margin-right: 12px; + border-color: ${(props) => props.theme.styles['primary-color']}; + color: ${(props) => props.theme.styles['primary-color']}; + letter-spacing: 2px; + &&:hover { + color: white; + background-color: ${(props) => props.theme.styles['primary-color']}; + border-color: ${(props) => props.theme.styles['primary-color']}; + } +`; + +const EditIconStyle = styled(EditFilled)` + && { + font-size: 12px; + } +`; + +type Props = { + state: DataContractState; + summary: AssertionStatusSummary; + showContractBuilder: () => void; +}; + +export const DataContractSummary = ({ state, summary, showContractBuilder }: Props) => { + const summaryIcon = getContractSummaryIcon(state, summary); + const summaryTitle = getContractSummaryTitle(state, summary); + const summaryMessage = getContractSummaryMessage(state, summary); + return ( + + + + {summaryIcon} + + {summaryTitle} + {summaryMessage} + + + + + + + EDIT + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractSummaryFooter.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractSummaryFooter.tsx new file mode 100644 index 0000000000000..6a892ebe2417a --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractSummaryFooter.tsx @@ -0,0 +1,75 @@ +import React from 'react'; +import styled from 'styled-components'; +import { ArrowRightOutlined } from '@ant-design/icons'; +import { Button } from 'antd'; +import { Assertion } from '../../../../../../../types.generated'; +import { StyledCheckOutlined, StyledCloseOutlined, StyledExclamationOutlined } from '../shared/styledComponents'; +import { getAssertionsSummary } from '../utils'; +import { ANTD_GRAY, REDESIGN_COLORS } from '../../../../constants'; + +const Container = styled.div` + display: flex; + align-items: center; + justify-content: space-between; +`; + +const StatusContainer = styled.div` + display: flex; + align-items: center; +`; + +const StatusText = styled.div` + color: ${ANTD_GRAY[8]}; + margin-left: 4px; +`; + +const ActionButton = styled(Button)` + color: ${REDESIGN_COLORS.BLUE}; +`; + +const StyledArrowRightOutlined = styled(ArrowRightOutlined)` + font-size: 8px; +`; + +type Props = { + assertions: Assertion[]; + passingText: string; + failingText: string; + errorText: string; + actionText?: string; + showAction?: boolean; +}; + +export const DataContractSummaryFooter = ({ + assertions, + actionText, + passingText, + errorText, + failingText, + showAction = true, +}: Props) => { + const summary = getAssertionsSummary(assertions); + const isFailing = summary.failing > 0; + const isPassing = summary.passing && summary.passing === summary.total; + const isErroring = summary.erroring > 0; + return ( + + + {(isFailing && ) || undefined} + {(isPassing && ) || undefined} + {(isErroring && !isFailing && ) || undefined} + + {(isFailing && failingText) || undefined} + {(isPassing && passingText) || undefined} + {(isErroring && errorText) || undefined} + + + {showAction && ( + + {actionText} + + + )} + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractTab.tsx new file mode 100644 index 0000000000000..52a7eca8730be --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractTab.tsx @@ -0,0 +1,117 @@ +import React, { useState } from 'react'; +import styled from 'styled-components'; +import { useGetDatasetContractQuery } from '../../../../../../../graphql/contract.generated'; +import { DataContractState } from '../../../../../../../types.generated'; +import { useEntityData } from '../../../../EntityContext'; +import { DataContractEmptyState } from './DataContractEmptyState'; +import { DataContractSummary } from './DataContractSummary'; +import { DataQualityContractSummary } from './DataQualityContractSummary'; +import { SchemaContractSummary } from './SchemaContractSummary'; +import { FreshnessContractSummary } from './FreshnessContractSummary'; +import { DataContractBuilderModal } from './builder/DataContractBuilderModal'; +import { createBuilderState } from './builder/utils'; +import { getAssertionsSummary } from '../utils'; + +const Container = styled.div` + display: flex; +`; + +const LeftColumn = styled.div` + width: 50%; +`; + +const RightColumn = styled.div` + width: 50%; +`; + +/** + * Component used for rendering the Data Contract Tab on the Assertions parent tab. + */ +export const DataContractTab = () => { + const { urn } = useEntityData(); + + const { data, refetch } = useGetDatasetContractQuery({ + variables: { + urn, + }, + }); + const [showContractBuilder, setShowContractBuilder] = useState(false); + + const contract = data?.dataset?.contract; + const schemaContracts = data?.dataset?.contract?.properties?.schema || []; + const freshnessContracts = data?.dataset?.contract?.properties?.freshness || []; + const dataQualityContracts = data?.dataset?.contract?.properties?.dataQuality || []; + const schemaAssertions = schemaContracts.map((c) => c.assertion); + const freshnessAssertions = freshnessContracts.map((c) => c.assertion); + const dataQualityAssertions = dataQualityContracts.map((c) => c.assertion); + const assertionsSummary = getAssertionsSummary([ + ...schemaAssertions, + ...freshnessAssertions, + ...dataQualityAssertions, + ] as any); + const contractState = data?.dataset?.contract?.status?.state || DataContractState.Active; + const hasFreshnessContract = freshnessContracts && freshnessContracts?.length; + const hasSchemaContract = schemaContracts && schemaContracts?.length; + const hasDataQualityContract = dataQualityContracts && dataQualityContracts?.length; + const showLeftColumn = hasFreshnessContract || hasSchemaContract || undefined; + + const onContractUpdate = () => { + if (contract) { + // Contract exists, just refetch. + refetch(); + } else { + // no contract yet, wait for indxing, + setTimeout(() => refetch(), 3000); + } + setShowContractBuilder(false); + }; + + return ( + <> + {data?.dataset?.contract ? ( + <> + setShowContractBuilder(true)} + /> + + {showLeftColumn && ( + + {(hasFreshnessContract && ( + + )) || + undefined} + {(hasSchemaContract && ( + + )) || + undefined} + + )} + + {hasDataQualityContract ? ( + + ) : undefined} + + + + ) : ( + setShowContractBuilder(true)} /> + )} + {showContractBuilder && ( + setShowContractBuilder(false)} + onSubmit={onContractUpdate} + /> + )} + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataQualityContractSummary.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataQualityContractSummary.tsx new file mode 100644 index 0000000000000..5b01540f859c6 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataQualityContractSummary.tsx @@ -0,0 +1,100 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Table } from 'antd'; +import { Assertion, DataQualityContract, DatasetAssertionInfo } from '../../../../../../../types.generated'; +import { ANTD_GRAY } from '../../../../constants'; +import { DataContractAssertionStatus } from './DataContractAssertionStatus'; +import { DataContractSummaryFooter } from './DataContractSummaryFooter'; +import { DatasetAssertionDescription } from '../DatasetAssertionDescription'; +import { FieldAssertionDescription } from '../FieldAssertionDescription'; +import { SqlAssertionDescription } from '../SqlAssertionDescription'; +import { VolumeAssertionDescription } from '../VolumeAssertionDescription'; + +const TitleText = styled.div` + color: ${ANTD_GRAY[7]}; + margin-bottom: 20px; + letter-spacing: 1px; +`; + +const ColumnHeader = styled.div` + color: ${ANTD_GRAY[8]}; +`; + +const Container = styled.div` + padding: 28px; +`; + +const SummaryContainer = styled.div` + width: 100%; + display: flex; + align-items: center; +`; + +const StyledTable = styled(Table)` + width: 100%; + border-radius: 8px; + box-shadow: 0px 0px 4px rgba(0, 0, 0, 0.1); +`; + +type Props = { + contracts: DataQualityContract[]; + showAction?: boolean; +}; + +export const DataQualityContractSummary = ({ contracts, showAction = false }: Props) => { + const assertions: Assertion[] = contracts?.map((contract) => contract.assertion); + + const columns = [ + { + title: () => ASSERTION, + render: (assertion: Assertion) => ( + <> + {assertion.info?.datasetAssertion && ( + + )} + {assertion.info?.volumeAssertion && ( + + )} + {assertion.info?.fieldAssertion && ( + + )} + {assertion.info?.sqlAssertion && } + + ), + }, + { + title: () => STATUS, + render: (assertion: Assertion) => , + }, + ]; + + const data = (assertions || []).map((assertion) => ({ + ...assertion, + key: assertion.urn, + })); + + return ( + + DATA QUALITY + + ( + + )} + /> + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessContractSummary.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessContractSummary.tsx new file mode 100644 index 0000000000000..efd0151b69bc2 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessContractSummary.tsx @@ -0,0 +1,96 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Divider } from 'antd'; +import { ClockCircleOutlined } from '@ant-design/icons'; +import { FreshnessContract } from '../../../../../../../types.generated'; +import { ANTD_GRAY } from '../../../../constants'; +import { DataContractSummaryFooter } from './DataContractSummaryFooter'; +import { FreshnessScheduleSummary } from './FreshnessScheduleSummary'; + +const Container = styled.div` + padding: 28px; +`; + +const TitleText = styled.div` + color: ${ANTD_GRAY[7]}; + margin-bottom: 20px; + letter-spacing: 1px; +`; + +const ThinDivider = styled(Divider)` + && { + padding: 0px; + margin: 0px; + } +`; + +const Header = styled.div` + color: ${ANTD_GRAY[8]}; + letter-spacing; 4px; + padding-top: 8px; + padding: 12px; + background-color: ${ANTD_GRAY[2]}; +`; + +const Body = styled.div` + padding: 12px; +`; + +const Footer = styled.div` + padding-top: 8px; + padding: 12px; + background-color: ${ANTD_GRAY[2]}; +`; + +const SummaryContainer = styled.div` + width: 100%; + border-radius: 8px; + box-shadow: 0px 0px 4px rgba(0, 0, 0, 0.1); +`; + +type Props = { + contracts: FreshnessContract[]; + showAction?: boolean; +}; + +export const FreshnessContractSummary = ({ contracts, showAction = false }: Props) => { + // TODO: Support multiple per-asset contracts. + const firstContract = (contracts.length && contracts[0]) || undefined; + const assertionDefinition = firstContract?.assertion?.info?.freshnessAssertion?.schedule; + const evaluationSchedule = (firstContract?.assertion as any)?.monitor?.relationships[0]?.entity?.info + ?.assertionMonitor?.assertions[0]?.schedule; + + return ( + + FRESHNESS + +
+ + UPDATE FREQUENCY +
+ + {!assertionDefinition && <>No contract found :(} + + {assertionDefinition && ( + + )} + + + +
+ +
+ + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessScheduleSummary.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessScheduleSummary.tsx new file mode 100644 index 0000000000000..434ccb985574f --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessScheduleSummary.tsx @@ -0,0 +1,28 @@ +import React from 'react'; +import cronstrue from 'cronstrue'; +import { + FreshnessAssertionSchedule, + FreshnessAssertionScheduleType, + CronSchedule, +} from '../../../../../../../types.generated'; +import { capitalizeFirstLetter } from '../../../../../../shared/textUtil'; + +type Props = { + definition: FreshnessAssertionSchedule; + evaluationSchedule?: CronSchedule; // When the assertion is run. +}; + +export const FreshnessScheduleSummary = ({ definition, evaluationSchedule }: Props) => { + const scheduleText = + definition.type === FreshnessAssertionScheduleType.Cron + ? `${capitalizeFirstLetter(cronstrue.toString(definition.cron?.cron as string))}.` + : `In the past ${ + definition.fixedInterval?.multiple + } ${definition.fixedInterval?.unit.toLocaleLowerCase()}s${ + (evaluationSchedule && + `, as of ${cronstrue.toString(evaluationSchedule.cron as string).toLowerCase()}`) || + '' + }`; + + return <>{scheduleText}; +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/SchemaContractSummary.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/SchemaContractSummary.tsx new file mode 100644 index 0000000000000..7313a1064634c --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/SchemaContractSummary.tsx @@ -0,0 +1,82 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Table } from 'antd'; +import { SchemaContract } from '../../../../../../../types.generated'; +import { ANTD_GRAY } from '../../../../constants'; +import { DataContractSummaryFooter } from './DataContractSummaryFooter'; + +const TitleText = styled.div` + color: ${ANTD_GRAY[7]}; + margin-bottom: 20px; + letter-spacing: 1px; +`; + +const ColumnHeader = styled.div` + color: ${ANTD_GRAY[8]}; +`; + +const Container = styled.div` + padding: 28px; +`; + +const SummaryContainer = styled.div` + width: 100%; + display: flex; + align-items: center; +`; + +const StyledTable = styled(Table)` + width: 100%; + border-radius: 8px; + box-shadow: 0px 0px 4px rgba(0, 0, 0, 0.1); + height: 100%; +`; + +type Props = { + contracts: SchemaContract[]; + showAction?: boolean; +}; + +export const SchemaContractSummary = ({ contracts, showAction = false }: Props) => { + const firstContract = (contracts.length && contracts[0]) || undefined; + const schemaMetadata = firstContract?.assertion?.info?.schemaAssertion?.schema; + + const columns = [ + { + title: () => NAME, + render: (field) => <>{field.fieldPath}, + }, + { + title: () => TYPE, + render: (field) => <>{field.type}, + }, + ]; + + const data = (schemaMetadata?.fields || []).map((field) => ({ + ...field, + key: field.fieldPath, + })); + + return ( + + SCHEMA + + ( + + )} + /> + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractAssertionGroupSelect.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractAssertionGroupSelect.tsx new file mode 100644 index 0000000000000..f96149dd0be5e --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractAssertionGroupSelect.tsx @@ -0,0 +1,54 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Assertion } from '../../../../../../../../types.generated'; +import { ANTD_GRAY } from '../../../../../constants'; +import { DataContractCategoryType } from './types'; +import { DatasetAssertionsList } from '../../DatasetAssertionsList'; + +const Category = styled.div` + padding: 20px; + font-weight: bold; + font-size: 14px; + background-color: ${ANTD_GRAY[3]}; + border-radius: 4px; +`; + +const Hint = styled.span` + font-weight: normal; + font-size: 14px; + color: ${ANTD_GRAY[8]}; +`; + +type Props = { + category: DataContractCategoryType; + multiple?: boolean; + assertions: Assertion[]; + selectedUrns: string[]; + onSelect: (assertionUrn: string) => void; +}; + +/** + * Used for selecting the assertions that are part of a data contract + */ +export const DataContractAssertionGroupSelect = ({ + category, + assertions, + multiple = true, + selectedUrns, + onSelect, +}: Props) => { + return ( + <> + + {category} {!multiple ? `(Choose 1)` : ''} + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractBuilder.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractBuilder.tsx new file mode 100644 index 0000000000000..0e6aef52a1cb7 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractBuilder.tsx @@ -0,0 +1,176 @@ +import React, { useState } from 'react'; +import { message, Button } from 'antd'; +import styled from 'styled-components'; +import lodash from 'lodash'; +import { DataContract, AssertionType, Assertion } from '../../../../../../../../types.generated'; +import { DataContractBuilderState, DataContractCategoryType, DEFAULT_BUILDER_STATE } from './types'; +import { buildUpsertDataContractMutationVariables } from './utils'; +import { useUpsertDataContractMutation } from '../../../../../../../../graphql/contract.generated'; +import { createAssertionGroups } from '../../utils'; +import { DataContractAssertionGroupSelect } from './DataContractAssertionGroupSelect'; +import { ANTD_GRAY } from '../../../../../constants'; +import { DATA_QUALITY_ASSERTION_TYPES } from '../utils'; +import { useGetDatasetAssertionsQuery } from '../../../../../../../../graphql/dataset.generated'; + +const AssertionsSection = styled.div` + border: 0.5px solid ${ANTD_GRAY[4]}; +`; + +const HeaderText = styled.div` + padding: 16px 20px; + color: ${ANTD_GRAY[7]}; + font-size: 16px; +`; + +const ActionContainer = styled.div` + display: flex; + justify-content: space-between; + margin-top: 16px; +`; + +const CancelButton = styled(Button)` + margin-left: 12px; +`; + +const SaveButton = styled(Button)` + margin-right: 20px; +`; + +type Props = { + entityUrn: string; + initialState?: DataContractBuilderState; + onSubmit?: (contract: DataContract) => void; + onCancel?: () => void; +}; + +/** + * This component is a modal used for constructing new Data Contracts + * + * In order to build a data contract, we simply list all dataset assertions and allow the user to choose. + */ +export const DataContractBuilder = ({ entityUrn, initialState, onSubmit, onCancel }: Props) => { + const isEdit = !!initialState; + const [builderState, setBuilderState] = useState(initialState || DEFAULT_BUILDER_STATE); + const [upsertDataContractMutation] = useUpsertDataContractMutation(); + + // note that for contracts, we do not allow the use of sibling node assertions, for clarity. + const { data } = useGetDatasetAssertionsQuery({ + variables: { urn: entityUrn }, + fetchPolicy: 'cache-first', + }); + const assertionData = data?.dataset?.assertions?.assertions ?? []; + + const assertionGroups = createAssertionGroups(assertionData as Array); + const freshnessAssertions = + assertionGroups.find((group) => group.type === AssertionType.Freshness)?.assertions || []; + const schemaAssertions = assertionGroups.find((group) => group.type === AssertionType.DataSchema)?.assertions || []; + const dataQualityAssertions = assertionGroups + .filter((group) => DATA_QUALITY_ASSERTION_TYPES.has(group.type)) + .flatMap((group) => group.assertions || []); + + /** + * Upserts the Data Contract for an entity + */ + const upsertDataContract = () => { + return upsertDataContractMutation({ + variables: buildUpsertDataContractMutationVariables(entityUrn, builderState), + }) + .then(({ data: dataContract, errors }) => { + if (!errors) { + message.success({ + content: isEdit ? `Edited Data Contract` : `Created Data Contract!`, + duration: 3, + }); + onSubmit?.(dataContract?.upsertDataContract as DataContract); + } + }) + .catch(() => { + message.destroy(); + message.error({ content: 'Failed to create Data Contract! An unexpected error occurred' }); + }); + }; + + const onSelectDataAssertion = (assertionUrn: string, type: string) => { + const selected = builderState[type]?.some((c) => c.assertionUrn === assertionUrn); + if (selected) { + setBuilderState({ + ...builderState, + [type]: builderState[type]?.filter((c) => c.assertionUrn !== assertionUrn), + }); + } else { + setBuilderState({ + ...builderState, + [type]: [...(builderState[type] || []), { assertionUrn }], + }); + } + }; + + const editDisabled = + lodash.isEqual(builderState, initialState) || lodash.isEqual(builderState, DEFAULT_BUILDER_STATE); + + const hasAssertions = freshnessAssertions.length || schemaAssertions.length || dataQualityAssertions.length; + + const onSelectFreshnessOrSchemaAssertion = (assertionUrn: string, type: string) => { + const selected = builderState[type]?.assertionUrn === assertionUrn; + if (selected) { + setBuilderState({ + ...builderState, + [type]: undefined, + }); + } else { + setBuilderState({ + ...builderState, + [type]: { assertionUrn }, + }); + } + }; + + return ( + <> + {(hasAssertions && Select the assertions that will make up your contract.) || ( + Add a few assertions on this entity to create a data contract out of them. + )} + + {(freshnessAssertions.length && ( + onSelectFreshnessOrSchemaAssertion(selectedUrn, 'freshness')} + /> + )) || + undefined} + {(schemaAssertions.length && ( + onSelectFreshnessOrSchemaAssertion(selectedUrn, 'schema')} + /> + )) || + undefined} + {(dataQualityAssertions.length && ( + c.assertionUrn) || []} + onSelect={(selectedUrn: string) => onSelectDataAssertion(selectedUrn, 'dataQuality')} + /> + )) || + undefined} + + + Cancel +
+ + Save + +
+ + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractBuilderModal.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractBuilderModal.tsx new file mode 100644 index 0000000000000..75a8fe0410918 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractBuilderModal.tsx @@ -0,0 +1,67 @@ +import React from 'react'; +import { Modal, Typography } from 'antd'; +import { DataContract } from '../../../../../../../../types.generated'; +import ClickOutside from '../../../../../../../shared/ClickOutside'; +import { DataContractBuilderState } from './types'; +import { DataContractBuilder } from './DataContractBuilder'; + +const modalStyle = {}; +const modalBodyStyle = { + paddingRight: 0, + paddingLeft: 0, + paddingBottom: 20, + paddingTop: 0, + maxHeight: '70vh', + 'overflow-x': 'auto', +}; + +type Props = { + entityUrn: string; + initialState?: DataContractBuilderState; + onSubmit?: (contract: DataContract) => void; + onCancel?: () => void; +}; + +/** + * This component is a modal used for constructing new Data Contracts + */ +export const DataContractBuilderModal = ({ entityUrn, initialState, onSubmit, onCancel }: Props) => { + const isEditing = initialState !== undefined; + const titleText = isEditing ? 'Edit Data Contract' : 'New Data Contract'; + + const modalClosePopup = () => { + Modal.confirm({ + title: 'Exit Editor', + content: `Are you sure you want to exit the editor? All changes will be lost`, + onOk() { + onCancel?.(); + }, + onCancel() {}, + okText: 'Yes', + maskClosable: true, + closable: true, + }); + }; + + return ( + + {titleText}} + style={modalStyle} + bodyStyle={modalBodyStyle} + visible + onCancel={onCancel} + > + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/types.ts b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/types.ts new file mode 100644 index 0000000000000..d527837efd72e --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/types.ts @@ -0,0 +1,37 @@ +/** + * The Data Contract Builder state + */ +export type DataContractBuilderState = { + /** + * The schema contract. In the UI, we only support defining a single schema contract. + */ + schema?: { + assertionUrn: string; + }; + + /** + * The freshness contract. In the UI, we only support defining a single freshness contract. + */ + freshness?: { + assertionUrn: string; + }; + + /** + * Data Quality contract. We cane define multiple data quality rules as part of the contract. + */ + dataQuality?: { + assertionUrn: string; + }[]; +}; + +export const DEFAULT_BUILDER_STATE = { + dataQuality: undefined, + schema: undefined, + freshness: undefined, +}; + +export enum DataContractCategoryType { + FRESHNESS = 'Freshness', + SCHEMA = 'Schema', + DATA_QUALITY = 'Data Quality', +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/utils.ts b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/utils.ts new file mode 100644 index 0000000000000..da2ae66d1ec9c --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/utils.ts @@ -0,0 +1,120 @@ +import { DataContract } from '../../../../../../../../types.generated'; +import { DataContractBuilderState, DataContractCategoryType } from './types'; + +/** + * Creates a builder state instance from a Data Contract object. + */ +export const createBuilderState = (contract?: DataContract | null): DataContractBuilderState | undefined => { + if (contract) { + return { + schema: + (contract?.properties?.schema?.length && { + assertionUrn: contract?.properties?.schema[0].assertion.urn, + }) || + undefined, + freshness: + (contract?.properties?.freshness?.length && { + assertionUrn: contract?.properties?.freshness[0].assertion.urn, + }) || + undefined, + dataQuality: + contract?.properties?.dataQuality?.map((c) => ({ assertionUrn: c.assertion.urn })) || undefined, + }; + } + return undefined; +}; + +/** + * Constructs the input variables required for upserting a data contract using graphql + */ +export const buildUpsertDataContractMutationVariables = (entityUrn: string, state: DataContractBuilderState) => { + return { + input: { + entityUrn, + freshness: (state.freshness && [state.freshness]) || [], + schema: (state.schema && [state.schema]) || [], + dataQuality: state.dataQuality || [], + }, + }; +}; + +/** + * Constructs the input variables required for removing an assertion from a data contract using graphql. + */ +export const buildRemoveAssertionFromContractMutationVariables = ( + entityUrn: string, + assertionUrn: string, + contract?: DataContract, +) => { + return { + input: { + entityUrn, + freshness: contract?.properties?.freshness + ?.filter((c) => c.assertion.urn !== assertionUrn) + ?.map((c) => ({ + assertionUrn: c.assertion.urn, + })), + schema: contract?.properties?.schema + ?.filter((c) => c.assertion.urn !== assertionUrn) + ?.map((c) => ({ + assertionUrn: c.assertion.urn, + })), + dataQuality: contract?.properties?.dataQuality + ?.filter((c) => c.assertion.urn !== assertionUrn) + ?.map((c) => ({ + assertionUrn: c.assertion.urn, + })), + }, + }; +}; + +/** + * Constructs the input variables required for adding an assertion to a data contract using graphql. + */ +export const buildAddAssertionToContractMutationVariables = ( + category: DataContractCategoryType, + entityUrn: string, + assertionUrn: string, + contract?: DataContract, +) => { + const baseInput = { + entityUrn, + freshness: contract?.properties?.freshness?.map((c) => ({ + assertionUrn: c.assertion.urn, + })), + schema: contract?.properties?.schema?.map((c) => ({ + assertionUrn: c.assertion.urn, + })), + dataQuality: contract?.properties?.dataQuality?.map((c) => ({ + assertionUrn: c.assertion.urn, + })), + }; + + switch (category) { + case DataContractCategoryType.SCHEMA: + // Replace schema assertion. We only support 1 schema assertion at a time (currently). + return { + input: { + ...baseInput, + schema: [{ assertionUrn }], + }, + }; + case DataContractCategoryType.FRESHNESS: + // Replace freshness assertion. We only support 1 freshness assertion at a time (currently). + return { + input: { + ...baseInput, + freshness: [{ assertionUrn }], + }, + }; + case DataContractCategoryType.DATA_QUALITY: + return { + input: { + ...baseInput, + dataQuality: [...(baseInput.dataQuality || []), { assertionUrn }], + }, + }; + default: + throw new Error(`Unrecognized category type ${category} provided.`); + } +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/utils.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/utils.tsx new file mode 100644 index 0000000000000..cc2e1bb7b386e --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/utils.tsx @@ -0,0 +1,114 @@ +import React from 'react'; +import { + CheckOutlined, + ClockCircleOutlined, + CloseOutlined, + ExclamationCircleFilled, + StopOutlined, +} from '@ant-design/icons'; +import { Assertion, AssertionType, DataContract, DataContractState } from '../../../../../../../types.generated'; +import { ANTD_GRAY } from '../../../../constants'; +import { FAILURE_COLOR_HEX, SUCCESS_COLOR_HEX, WARNING_COLOR_HEX } from '../utils'; +import { DataContractCategoryType } from './builder/types'; +import { AssertionStatusSummary } from '../types'; + +export const getContractSummaryIcon = (state: DataContractState, summary: AssertionStatusSummary) => { + if (state === DataContractState.Pending) { + return ; + } + if (summary.total === 0) { + return ; + } + if (summary.passing === summary.total) { + return ; + } + if (summary.failing > 0) { + return ; + } + if (summary.erroring > 0) { + return ; + } + return ; +}; + +export const getContractSummaryTitle = (state: DataContractState, summary: AssertionStatusSummary) => { + if (state === DataContractState.Pending) { + return 'This contract is pending implementation'; + } + if (summary.total === 0) { + return 'This contract has not yet been validated'; + } + if (summary.passing === summary.total) { + return 'This dataset is meeting its contract'; + } + if (summary.failing > 0) { + return 'This dataset is not meeting its contract'; + } + if (summary.erroring > 0) { + return 'Unable to determine contract status'; + } + return 'This contract has not yet been validated'; +}; + +export const getContractSummaryMessage = (state: DataContractState, summary: AssertionStatusSummary) => { + if (state === DataContractState.Pending) { + return 'This may take some time. Come back later!'; + } + if (summary.total === 0) { + return 'No contract assertions have been run yet'; + } + if (summary.passing === summary.total) { + return 'All contract assertions are passing'; + } + if (summary.failing > 0) { + return 'Some contract assertions are failing'; + } + if (summary.erroring > 0) { + return 'Some contract assertions are completing with errors'; + } + return 'No contract assertions have been run yet'; +}; + +/** + * Returns true if a given assertion is part of a given contract, false otherwise. + */ +export const isAssertionPartOfContract = (assertion: Assertion, contract: DataContract) => { + if (contract.properties?.dataQuality?.some((c) => c.assertion.urn === assertion.urn)) { + return true; + } + if (contract.properties?.schema?.some((c) => c.assertion.urn === assertion.urn)) { + return true; + } + if (contract.properties?.freshness?.some((c) => c.assertion.urn === assertion.urn)) { + return true; + } + return false; +}; + +/** + * Retrieves the high level contract category - schema, freshness, or data quality - given an assertion + */ +export const getDataContractCategoryFromAssertion = (assertion: Assertion) => { + if ( + assertion.info?.type === AssertionType.Dataset || + assertion.info?.type === AssertionType.Volume || + assertion.info?.type === AssertionType.Field || + assertion.info?.type === AssertionType.Sql + ) { + return DataContractCategoryType.DATA_QUALITY; + } + if (assertion.info?.type === AssertionType.Freshness) { + return DataContractCategoryType.FRESHNESS; + } + if (assertion.info?.type === AssertionType.DataSchema) { + return DataContractCategoryType.SCHEMA; + } + return DataContractCategoryType.DATA_QUALITY; +}; + +export const DATA_QUALITY_ASSERTION_TYPES = new Set([ + AssertionType.Volume, + AssertionType.Sql, + AssertionType.Field, + AssertionType.Dataset, +]); diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/fieldDescriptionUtils.ts b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/fieldDescriptionUtils.ts new file mode 100644 index 0000000000000..3c6e14f1d80ab --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/fieldDescriptionUtils.ts @@ -0,0 +1,179 @@ +import { + AssertionStdOperator, + AssertionStdParameters, + FieldAssertionInfo, + FieldAssertionType, + FieldMetricType, + FieldTransformType, +} from '../../../../../../types.generated'; +import { formatNumberWithoutAbbreviation } from '../../../../../shared/formatNumber'; +import { parseMaybeStringAsFloatOrDefault } from '../../../../../shared/numberUtil'; + +const ASSERTION_OPERATOR_TO_DESCRIPTION: Record = { + [AssertionStdOperator.EqualTo]: 'Is equal to', + [AssertionStdOperator.NotEqualTo]: 'Is not equal to', + [AssertionStdOperator.Contain]: 'Contains', + [AssertionStdOperator.RegexMatch]: 'Matches', + [AssertionStdOperator.StartWith]: 'Starts with', + [AssertionStdOperator.EndWith]: 'Ends with', + [AssertionStdOperator.In]: 'Is in', + [AssertionStdOperator.NotIn]: 'Is not in', + + [AssertionStdOperator.IsFalse]: 'Is False', + [AssertionStdOperator.IsTrue]: 'Is True', + [AssertionStdOperator.Null]: 'Is NULL', + [AssertionStdOperator.NotNull]: 'Is not NULL', + + [AssertionStdOperator.GreaterThan]: 'Is greater than', + [AssertionStdOperator.GreaterThanOrEqualTo]: 'Is greater than or equal to', + [AssertionStdOperator.LessThan]: 'Is less than', + [AssertionStdOperator.LessThanOrEqualTo]: 'Is less than or equal to', + [AssertionStdOperator.Between]: 'Is within a range', + + [AssertionStdOperator.Native]: undefined, +}; + +const SUPPORTED_OPERATORS_FOR_FIELD_DESCRIPTION = [ + AssertionStdOperator.EqualTo, + AssertionStdOperator.Null, + AssertionStdOperator.NotNull, + AssertionStdOperator.NotEqualTo, + AssertionStdOperator.NotIn, + AssertionStdOperator.RegexMatch, + AssertionStdOperator.GreaterThan, + AssertionStdOperator.LessThan, + AssertionStdOperator.GreaterThanOrEqualTo, + AssertionStdOperator.LessThanOrEqualTo, + AssertionStdOperator.In, + AssertionStdOperator.Between, + AssertionStdOperator.Contain, + AssertionStdOperator.IsTrue, + AssertionStdOperator.IsFalse, +]; + +const getAssertionStdOperator = (operator: AssertionStdOperator) => { + if (!ASSERTION_OPERATOR_TO_DESCRIPTION[operator] || !SUPPORTED_OPERATORS_FOR_FIELD_DESCRIPTION.includes(operator)) { + throw new Error(`Unknown operator ${operator}`); + } + return ASSERTION_OPERATOR_TO_DESCRIPTION[operator]?.toLowerCase(); +}; + +export const getFieldMetricTypeReadableLabel = (metric: FieldMetricType) => { + switch (metric) { + case FieldMetricType.NullCount: + return 'Null count'; + case FieldMetricType.NullPercentage: + return 'Null percentage'; + case FieldMetricType.UniqueCount: + return 'Unique count'; + case FieldMetricType.UniquePercentage: + return 'Unique percentage'; + case FieldMetricType.MaxLength: + return 'Max length'; + case FieldMetricType.MinLength: + return 'Min length'; + case FieldMetricType.EmptyCount: + return 'Empty count'; + case FieldMetricType.EmptyPercentage: + return 'Empty percentage'; + case FieldMetricType.Max: + return 'Max'; + case FieldMetricType.Min: + return 'Min'; + case FieldMetricType.Mean: + return 'Average'; + case FieldMetricType.Median: + return 'Median'; + case FieldMetricType.NegativeCount: + return 'Negative count'; + case FieldMetricType.NegativePercentage: + return 'Negative percentage'; + case FieldMetricType.Stddev: + return 'Standard deviation'; + case FieldMetricType.ZeroCount: + return 'Zero count'; + case FieldMetricType.ZeroPercentage: + return 'Zero percentage'; + default: + throw new Error(`Unknown field metric type ${metric}`); + } +}; + +const getFieldTransformType = (transform: FieldTransformType) => { + switch (transform) { + case FieldTransformType.Length: + return 'Length'; + default: + throw new Error(`Unknown field transform type ${transform}`); + } +}; + +const getAssertionStdParameters = (parameters: AssertionStdParameters) => { + if (parameters.value) { + return formatNumberWithoutAbbreviation( + parseMaybeStringAsFloatOrDefault(parameters.value.value, parameters.value.value), + ); + } + if (parameters.minValue && parameters.maxValue) { + return `${formatNumberWithoutAbbreviation( + parseMaybeStringAsFloatOrDefault(parameters.minValue.value, parameters.minValue.value), + )} and ${formatNumberWithoutAbbreviation( + parseMaybeStringAsFloatOrDefault(parameters.maxValue.value, parameters.maxValue.value), + )}`; + } + return ''; +}; + +export const getFieldDescription = (assertionInfo: FieldAssertionInfo) => { + const { type, fieldValuesAssertion, fieldMetricAssertion } = assertionInfo; + switch (type) { + case FieldAssertionType.FieldValues: + return fieldValuesAssertion?.field?.path; + case FieldAssertionType.FieldMetric: + return fieldMetricAssertion?.field?.path; + default: + throw new Error(`Unknown field assertion type ${type}`); + } +}; + +export const getFieldOperatorDescription = (assertionInfo: FieldAssertionInfo) => { + const { type, fieldValuesAssertion, fieldMetricAssertion } = assertionInfo; + switch (type) { + case FieldAssertionType.FieldValues: + if (!fieldValuesAssertion?.operator) return ''; + return getAssertionStdOperator(fieldValuesAssertion.operator); + case FieldAssertionType.FieldMetric: + if (!fieldMetricAssertion?.operator) return ''; + return getAssertionStdOperator(fieldMetricAssertion.operator); + default: + throw new Error(`Unknown field assertion type ${type}`); + } +}; + +export const getFieldTransformDescription = (assertionInfo: FieldAssertionInfo) => { + const { type, fieldValuesAssertion, fieldMetricAssertion } = assertionInfo; + switch (type) { + case FieldAssertionType.FieldValues: + if (!fieldValuesAssertion?.transform?.type) return ''; + return getFieldTransformType(fieldValuesAssertion.transform.type); + case FieldAssertionType.FieldMetric: + if (!fieldMetricAssertion?.metric) return ''; + return getFieldMetricTypeReadableLabel(fieldMetricAssertion.metric); + default: + throw new Error(`Unknown field assertion type ${type}`); + } +}; + +export const getFieldParametersDescription = (assertionInfo: FieldAssertionInfo) => { + const { type, fieldValuesAssertion, fieldMetricAssertion } = assertionInfo; + switch (type) { + case FieldAssertionType.FieldValues: + if (!fieldValuesAssertion?.parameters) return ''; + return getAssertionStdParameters(fieldValuesAssertion.parameters); + case FieldAssertionType.FieldMetric: + if (!fieldMetricAssertion?.parameters) return ''; + return getAssertionStdParameters(fieldMetricAssertion.parameters); + default: + throw new Error(`Unknown field assertion type ${type}`); + } +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/shared/styledComponents.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/shared/styledComponents.tsx new file mode 100644 index 0000000000000..14651899a355f --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/shared/styledComponents.tsx @@ -0,0 +1,32 @@ +import styled from 'styled-components'; +import { CheckOutlined, ClockCircleOutlined, CloseOutlined, ExclamationCircleOutlined } from '@ant-design/icons'; +import { ANTD_GRAY } from '../../../../constants'; +import { FAILURE_COLOR_HEX, SUCCESS_COLOR_HEX, WARNING_COLOR_HEX } from '../utils'; + +export const StyledCheckOutlined = styled(CheckOutlined)` + color: ${SUCCESS_COLOR_HEX}; + font-size: 16px; + margin-right: 4px; + margin-left: 4px; +`; + +export const StyledCloseOutlined = styled(CloseOutlined)` + color: ${FAILURE_COLOR_HEX}; + font-size: 16px; + margin-right: 4px; + margin-left: 4px; +`; + +export const StyledExclamationOutlined = styled(ExclamationCircleOutlined)` + color: ${WARNING_COLOR_HEX}; + font-size: 16px; + margin-right: 4px; + margin-left: 4px; +`; + +export const StyledClockCircleOutlined = styled(ClockCircleOutlined)` + color: ${ANTD_GRAY[6]}; + font-size: 16px; + margin-right: 4px; + margin-left: 4px; +`; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/types.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/types.tsx new file mode 100644 index 0000000000000..8a70a3d87c147 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/types.tsx @@ -0,0 +1,21 @@ +import { Assertion, AssertionType } from '../../../../../../types.generated'; + +export type AssertionStatusSummary = { + passing: number; + failing: number; + erroring: number; + total: number; // Total assertions with at least 1 run. + totalAssertions: number; +}; + +/** + * A group of assertions related by their logical type or category. + */ +export type AssertionGroup = { + name: string; + icon: React.ReactNode; + description?: string; + assertions: Assertion[]; + summary: AssertionStatusSummary; + type: AssertionType; +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/utils.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/utils.tsx new file mode 100644 index 0000000000000..019271b4732a1 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/utils.tsx @@ -0,0 +1,378 @@ +import React from 'react'; +import styled from 'styled-components'; +import { + ClockCircleOutlined, + TableOutlined, + ProjectOutlined, + ConsoleSqlOutlined, + CheckOutlined, + CloseOutlined, + ApiOutlined, + CodeOutlined, + ExclamationCircleOutlined, +} from '@ant-design/icons'; +import { + Assertion, + AssertionResultType, + AssertionStdOperator, + AssertionStdParameters, + AssertionType, + AssertionValueChangeType, + EntityType, + VolumeAssertionInfo, + VolumeAssertionType, +} from '../../../../../../types.generated'; +import { sortAssertions } from './assertionUtils'; +import { AssertionGroup, AssertionStatusSummary } from './types'; +import { formatNumberWithoutAbbreviation } from '../../../../../shared/formatNumber'; +import { parseMaybeStringAsFloatOrDefault } from '../../../../../shared/numberUtil'; + +export const SUCCESS_COLOR_HEX = '#52C41A'; +export const FAILURE_COLOR_HEX = '#F5222D'; +export const WARNING_COLOR_HEX = '#FA8C16'; + +const StyledClockCircleOutlined = styled(ClockCircleOutlined)` + && { + margin: 0px; + padding: 0px; + margin-right: 8px; + font-size: 14px; + } +`; + +const StyledTableOutlined = styled(TableOutlined)` + && { + margin: 0px; + padding: 0px; + margin-right: 8px; + font-size: 18px; + } +`; + +const StyledProjectOutlined = styled(ProjectOutlined)` + && { + margin: 0px; + padding: 0px; + margin-right: 8px; + font-size: 18px; + } +`; + +const StyledConsoleSqlOutlined = styled(ConsoleSqlOutlined)` + && { + margin: 0px; + padding: 0px; + margin-right: 8px; + font-size: 18px; + } +`; + +const StyledApiOutlined = styled(ApiOutlined)` + && { + margin: 0px; + padding: 0px; + margin-right: 8px; + font-size: 18px; + } +`; + +const StyledCheckOutlined = styled(CheckOutlined)` + && { + color: ${SUCCESS_COLOR_HEX}; + font-size: 14px; + padding: 0px; + margin: 0px; + } +`; + +const StyledCloseOutlined = styled(CloseOutlined)` + && { + color: ${FAILURE_COLOR_HEX}; + font-size: 14px; + padding: 0px; + margin: 0px; + } +`; + +const StyledExclamationOutlined = styled(ExclamationCircleOutlined)` + && { + color: ${WARNING_COLOR_HEX}; + font-size: 14px; + padding: 0px; + margin: 0px; + } +`; + +const StyledCodeOutlined = styled(CodeOutlined)` + && { + margin: 0px; + padding: 0px; + margin-right: 8px; + font-size: 18px; + } +`; + +export const ASSERTION_INFO = [ + { + name: 'Freshness', + description: 'Define & monitor your expectations about when this dataset should be updated', + icon: , + type: AssertionType.Freshness, + entityTypes: [EntityType.Dataset], + enabled: true, + visible: true, + }, + { + name: 'Volume', + description: 'Define & monitor your expectations about the size of this dataset', + icon: , + type: AssertionType.Volume, + entityTypes: [EntityType.Dataset], + enabled: true, + visible: true, + }, + { + name: 'Column', + description: 'Define & monitor your expectations about the values in a column', + icon: , + type: AssertionType.Field, + entityTypes: [EntityType.Dataset], + enabled: true, + visible: true, + requiresConnectionSupportedByMonitors: false, + }, + { + name: 'Schema', + description: "Define & monitor your expectations about the table's columns and their types", + icon: , + type: AssertionType.DataSchema, + entityTypes: [EntityType.Dataset], + enabled: true, + visible: true, + }, + { + name: 'Custom', + description: 'Define & monitor your expectations using custom SQL rules', + icon: , + type: AssertionType.Sql, + entityTypes: [EntityType.Dataset], + enabled: true, + visible: true, + requiresConnectionSupportedByMonitors: true, + }, + { + name: 'Other', + description: 'Assertions that are defined and maintained outside of DataHub.', + icon: , + type: AssertionType.Dataset, + entityTypes: [EntityType.Dataset], + enabled: false, + visible: false, + }, +]; + +const ASSERTION_TYPE_TO_INFO = new Map(); +ASSERTION_INFO.forEach((info) => { + ASSERTION_TYPE_TO_INFO.set(info.type, info); +}); + +const getAssertionGroupName = (type: AssertionType): string => { + return ASSERTION_TYPE_TO_INFO.has(type) ? ASSERTION_TYPE_TO_INFO.get(type).name : 'Unknown'; +}; + +const getAssertionGroupTypeIcon = (type: AssertionType) => { + return ASSERTION_TYPE_TO_INFO.has(type) ? ASSERTION_TYPE_TO_INFO.get(type).icon : undefined; +}; + +/** + * Returns a status summary for the assertions associated with a Dataset. + * + * @param assertions The assertions to extract the summary for + */ +export const getAssertionsSummary = (assertions: Assertion[]): AssertionStatusSummary => { + const summary = { + passing: 0, + failing: 0, + erroring: 0, + total: 0, + totalAssertions: assertions.length, + }; + assertions.forEach((assertion) => { + if ((assertion.runEvents?.runEvents?.length || 0) > 0) { + const mostRecentRun = assertion.runEvents?.runEvents?.[0]; + const resultType = mostRecentRun?.result?.type; + if (AssertionResultType.Success === resultType) { + summary.passing++; + } + if (AssertionResultType.Failure === resultType) { + summary.failing++; + } + if (AssertionResultType.Error === resultType) { + summary.erroring++; + } + if (AssertionResultType.Init !== resultType) { + summary.total++; // only count assertions for which there is one completed run event, ignoring INIT statuses! + } + } + }); + return summary; +}; + +// /** +// * Returns a list of assertion groups, where assertions are grouped +// * by their "type" or "category". Each group includes the assertions inside, along with +// * a summary of passing and failing assertions for the group. +// * +// * @param assertions The assertions to group +// */ +export const createAssertionGroups = (assertions: Array): AssertionGroup[] => { + // Pre-sort the list of assertions based on which has been most recently executed. + const newAssertions = [...assertions].sort(sortAssertions); + + const typeToAssertions = new Map(); + newAssertions + .filter((assertion) => assertion.info?.type) + .forEach((assertion) => { + const groupType = assertion.info?.type; + const groupedAssertions = typeToAssertions.get(groupType) || []; + groupedAssertions.push(assertion); + typeToAssertions.set(groupType, groupedAssertions); + }); + + // Now, create summary for each type and build the AssertionGroup object + const assertionGroups: AssertionGroup[] = []; + typeToAssertions.forEach((groupedAssertions, type) => { + const newGroup: AssertionGroup = { + name: getAssertionGroupName(type), + icon: getAssertionGroupTypeIcon(type), + assertions: groupedAssertions, + summary: getAssertionsSummary(groupedAssertions), + type, + }; + assertionGroups.push(newGroup); + }); + + return assertionGroups; +}; + +export const getAssertionGroupSummaryIcon = (summary: AssertionStatusSummary) => { + if (summary.total === 0) { + return null; + } + if (summary.passing === summary.total) { + return ; + } + if (summary.erroring > 0) { + return ; + } + return ; +}; + +export const getAssertionGroupSummaryMessage = (summary: AssertionStatusSummary) => { + if (summary.total === 0) { + return 'No assertions have run'; + } + if (summary.passing === summary.total) { + return 'All assertions are passing'; + } + if (summary.erroring > 0) { + return 'An error is preventing some assertions from running'; + } + if (summary.failing === summary.total) { + return 'All assertions are failing'; + } + return 'Some assertions are failing'; +}; + +export const getAssertionTypesForEntityType = (entityType: EntityType, monitorsConnectionForEntityExists: boolean) => { + return ASSERTION_INFO.filter((type) => type.entityTypes.includes(entityType)).map((type) => ({ + ...type, + enabled: type.enabled && (!type.requiresConnectionSupportedByMonitors || monitorsConnectionForEntityExists), + })); +}; + +type VolumeTypeField = + | 'rowCountTotal' + | 'rowCountChange' + | 'incrementingSegmentRowCountTotal' + | 'incrementingSegmentRowCountChange'; + +export const getPropertyFromVolumeType = (type: VolumeAssertionType) => { + switch (type) { + case VolumeAssertionType.RowCountTotal: + return 'rowCountTotal' as VolumeTypeField; + case VolumeAssertionType.RowCountChange: + return 'rowCountChange' as VolumeTypeField; + case VolumeAssertionType.IncrementingSegmentRowCountTotal: + return 'incrementingSegmentRowCountTotal' as VolumeTypeField; + case VolumeAssertionType.IncrementingSegmentRowCountChange: + return 'incrementingSegmentRowCountChange' as VolumeTypeField; + default: + throw new Error(`Unknown volume assertion type: ${type}`); + } +}; + +export const getVolumeTypeInfo = (volumeAssertion: VolumeAssertionInfo) => { + const result = volumeAssertion[getPropertyFromVolumeType(volumeAssertion.type)]; + if (!result) { + return undefined; + } + return result; +}; + +export const getIsRowCountChange = (type: VolumeAssertionType) => { + return [VolumeAssertionType.RowCountChange, VolumeAssertionType.IncrementingSegmentRowCountChange].includes(type); +}; + +export const getVolumeTypeDescription = (volumeType: VolumeAssertionType) => { + switch (volumeType) { + case VolumeAssertionType.RowCountTotal: + case VolumeAssertionType.IncrementingSegmentRowCountTotal: + return 'has'; + case VolumeAssertionType.RowCountChange: + case VolumeAssertionType.IncrementingSegmentRowCountChange: + return 'should grow by'; + default: + throw new Error(`Unknown volume type ${volumeType}`); + } +}; + +export const getOperatorDescription = (operator: AssertionStdOperator) => { + switch (operator) { + case AssertionStdOperator.GreaterThanOrEqualTo: + return 'at least'; + case AssertionStdOperator.LessThanOrEqualTo: + return 'at most'; + case AssertionStdOperator.Between: + return 'between'; + default: + throw new Error(`Unknown operator ${operator}`); + } +}; + +export const getValueChangeTypeDescription = (valueChangeType: AssertionValueChangeType) => { + switch (valueChangeType) { + case AssertionValueChangeType.Absolute: + return 'rows'; + case AssertionValueChangeType.Percentage: + return '%'; + default: + throw new Error(`Unknown value change type ${valueChangeType}`); + } +}; + +export const getParameterDescription = (parameters: AssertionStdParameters) => { + if (parameters.value) { + return formatNumberWithoutAbbreviation( + parseMaybeStringAsFloatOrDefault(parameters.value.value, parameters.value.value), + ); + } + if (parameters.minValue && parameters.maxValue) { + return `${formatNumberWithoutAbbreviation( + parseMaybeStringAsFloatOrDefault(parameters.minValue.value, parameters.minValue.value), + )} and ${formatNumberWithoutAbbreviation( + parseMaybeStringAsFloatOrDefault(parameters.maxValue.value, parameters.maxValue.value), + )}`; + } + throw new Error('Invalid assertion parameters provided'); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/EditorTheme.tsx b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/EditorTheme.tsx index ec094fb84e59a..b3cc3c6a225e6 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/EditorTheme.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/EditorTheme.tsx @@ -81,7 +81,7 @@ export const EditorContainer = styled.div<{ editorStyle?: string }>` line-height: 1.5; white-space: pre-wrap; margin: 0; - ${props => props.editorStyle} + ${(props) => props.editorStyle} a { font-weight: 500; diff --git a/datahub-web-react/src/app/entity/shared/tabs/ERModelRelationship/ERModelRelationshipTab.less b/datahub-web-react/src/app/entity/shared/tabs/ERModelRelationship/ERModelRelationshipTab.less index bb8f2f04f6a37..897dbf251330b 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/ERModelRelationship/ERModelRelationshipTab.less +++ b/datahub-web-react/src/app/entity/shared/tabs/ERModelRelationship/ERModelRelationshipTab.less @@ -1,4 +1,4 @@ -@import "../../../../../../node_modules/antd/dist/antd.less"; +@import '../../../../../../node_modules/antd/dist/antd.less'; .ERModelRelationTab { .add-btn-link { @@ -12,19 +12,17 @@ font-size: 12px !important; line-height: 20px !important; color: #262626 !important; - } - .thin-divider{ + .thin-divider { width: 100%; - height:0px; - border:1px solid rgba(0,0,0,0.12) !important; - flex:none; - order:5; - align-self:stretch; - flex-grow:0; + height: 0px; + border: 1px solid rgba(0, 0, 0, 0.12) !important; + flex: none; + order: 5; + align-self: stretch; + flex-grow: 0; } .search-header-div { - padding: 8px 20px; gap: 238px; @@ -34,11 +32,11 @@ width: 1292px !important; height: 56px !important; - background: #FFFFFF; + background: #ffffff; box-shadow: 0px 2px 6px rgba(0, 0, 0, 0.05); } .ermodelrelation-preview-div { padding-left: 22px; padding-top: 34px; } -} \ No newline at end of file +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/ERModelRelationship/ERModelRelationshipTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/ERModelRelationship/ERModelRelationshipTab.tsx index 946bf429d3e87..e4869356f7661 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/ERModelRelationship/ERModelRelationshipTab.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/ERModelRelationship/ERModelRelationshipTab.tsx @@ -8,7 +8,13 @@ export const ERModelRelationshipTab = () => { const { entityData } = useEntityData(); const refetch = useRefetch(); const ermodelrelationView = (ermodelrelationData?: any): JSX.Element => { - return ; + return ( + + ); }; return ( <> diff --git a/datahub-web-react/src/app/entity/shared/tabs/Incident/components/IncidentListItem.tsx b/datahub-web-react/src/app/entity/shared/tabs/Incident/components/IncidentListItem.tsx index 2fef9de5c3166..4c9f83ab76621 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Incident/components/IncidentListItem.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Incident/components/IncidentListItem.tsx @@ -230,13 +230,14 @@ export default function IncidentListItem({ incident, refetch }: Props) { Description {incident?.description} - {incident.status.state === IncidentState.Resolved ? + {incident.status.state === IncidentState.Resolved ? ( <> Resolution Note - {incident?.status.message || 'No additional details'} + + {incident?.status.message || 'No additional details'} + - : null - } + ) : null} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Lineage/ImpactAnalysis.tsx b/datahub-web-react/src/app/entity/shared/tabs/Lineage/ImpactAnalysis.tsx index 4f1c5bb98807d..7eeff26cc815c 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Lineage/ImpactAnalysis.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Lineage/ImpactAnalysis.tsx @@ -27,7 +27,7 @@ export const ImpactAnalysis = ({ setSkipCache, resetShouldRefetch, onLineageClick, - isLineageTab + isLineageTab, }: Props) => { const finalStartTimeMillis = startTimeMillis || undefined; const finalEndTimeMillis = endTimeMillis || undefined; diff --git a/datahub-web-react/src/app/entity/user/UserInfoSideBar.tsx b/datahub-web-react/src/app/entity/user/UserInfoSideBar.tsx index 71bfbfcd49a16..4e3dc45489b55 100644 --- a/datahub-web-react/src/app/entity/user/UserInfoSideBar.tsx +++ b/datahub-web-react/src/app/entity/user/UserInfoSideBar.tsx @@ -62,17 +62,18 @@ export default function UserInfoSideBar({ sideBarData, refetch }: Props) { const me = useUserContext(); const isProfileOwner = me?.user?.urn === urn; - const { updateTitle } = useBrowserTitle(); - - useEffect(()=>{ + const { updateTitle } = useBrowserTitle(); + + useEffect(() => { // You can use the title and updateTitle function here // For example, updating the title when the component mounts - if(name){ + if (name) { updateTitle(`User | ${name}`); } // // Don't forget to clean up the title when the component unmounts return () => { - if(name){ // added to condition for rerendering issue + if (name) { + // added to condition for rerendering issue updateTitle(''); } }; diff --git a/datahub-web-react/src/app/entity/user/UserProfile.tsx b/datahub-web-react/src/app/entity/user/UserProfile.tsx index ffbfbeb977527..2d6efb95b9d2a 100644 --- a/datahub-web-react/src/app/entity/user/UserProfile.tsx +++ b/datahub-web-react/src/app/entity/user/UserProfile.tsx @@ -119,7 +119,7 @@ export default function UserProfile() { if (data?.corpUser?.exists === false) { return ; } - + return ( <> {error && } diff --git a/datahub-web-react/src/app/glossary/GlossaryBrowser/NodeItem.tsx b/datahub-web-react/src/app/glossary/GlossaryBrowser/NodeItem.tsx index cd6593e2d2f07..af3c7454425de 100644 --- a/datahub-web-react/src/app/glossary/GlossaryBrowser/NodeItem.tsx +++ b/datahub-web-react/src/app/glossary/GlossaryBrowser/NodeItem.tsx @@ -166,7 +166,12 @@ function NodeItem(props: Props) { ))} {!hideTerms && (childTerms as GlossaryTerm[]).map((child) => ( - + ))} )} diff --git a/datahub-web-react/src/app/glossary/GlossarySearch.tsx b/datahub-web-react/src/app/glossary/GlossarySearch.tsx index 321c218c38fe3..160f8e66d709d 100644 --- a/datahub-web-react/src/app/glossary/GlossarySearch.tsx +++ b/datahub-web-react/src/app/glossary/GlossarySearch.tsx @@ -89,7 +89,6 @@ function GlossarySearch() { onFocus={() => setIsSearchBarFocused(true)} /> {isSearchBarFocused && searchResults && !!searchResults.length && renderSearchResults()} - ); diff --git a/datahub-web-react/src/app/home/HomePageRecommendations.tsx b/datahub-web-react/src/app/home/HomePageRecommendations.tsx index fa8da01e8079b..c55c194291d94 100644 --- a/datahub-web-react/src/app/home/HomePageRecommendations.tsx +++ b/datahub-web-react/src/app/home/HomePageRecommendations.tsx @@ -117,7 +117,7 @@ export const HomePageRecommendations = ({ user }: Props) => { variables: { input: { types: browseEntityList, - viewUrn + viewUrn, }, }, }); @@ -138,7 +138,7 @@ export const HomePageRecommendations = ({ user }: Props) => { scenario, }, limit: 10, - viewUrn + viewUrn, }, }, fetchPolicy: 'no-cache', @@ -191,9 +191,8 @@ export const HomePageRecommendations = ({ user }: Props) => { {orderedEntityCounts.map( (entityCount) => entityCount && - entityCount.count !== 0 && - entityCount.entityType !== EntityType.BusinessAttribute && - ( + entityCount.count !== 0 && + entityCount.entityType !== EntityType.BusinessAttribute && ( { {orderedEntityCounts.map( (entityCount) => entityCount && - entityCount.count !== 0 && - entityCount.entityType === EntityType.BusinessAttribute && + entityCount.count !== 0 && + entityCount.entityType === EntityType.BusinessAttribute && businessAttributesFlag && ( { (entityCount) => entityCount.entityType === EntityType.GlossaryTerm, ) && } - ) : ( + ) : ( - )} + )} )} {recommendationModules && diff --git a/datahub-web-react/src/app/identity/group/GroupListItem.tsx b/datahub-web-react/src/app/identity/group/GroupListItem.tsx index e5aada4800253..3559c4bdbb42c 100644 --- a/datahub-web-react/src/app/identity/group/GroupListItem.tsx +++ b/datahub-web-react/src/app/identity/group/GroupListItem.tsx @@ -54,7 +54,11 @@ export default function GroupListItem({ group, onDelete, selectRoleOptions, refe - +
{displayName} diff --git a/datahub-web-react/src/app/identity/user/UserListItem.tsx b/datahub-web-react/src/app/identity/user/UserListItem.tsx index 8ad3d7d93d657..ff349664d7628 100644 --- a/datahub-web-react/src/app/identity/user/UserListItem.tsx +++ b/datahub-web-react/src/app/identity/user/UserListItem.tsx @@ -98,8 +98,8 @@ export default function UserListItem({ user, canManageUserCredentials, selectRol
{displayName}
-
- {user.username} +
+ {user.username}
{userStatus && ( diff --git a/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx b/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx index baa92cbb7d46c..1d04edbac228a 100644 --- a/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx +++ b/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx @@ -58,9 +58,9 @@ export const ManageIngestionPage = () => { - Manage Ingestion + Manage Data Sources - Create, schedule, and run DataHub ingestion sources. + Configure and schedule syncs to import data from your data sources onClickTab(tab)}> diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceTable.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceTable.tsx index ad1e7f6425062..00d04ed245edf 100644 --- a/datahub-web-react/src/app/ingest/source/IngestionSourceTable.tsx +++ b/datahub-web-react/src/app/ingest/source/IngestionSourceTable.tsx @@ -3,7 +3,7 @@ import React from 'react'; import styled from 'styled-components/macro'; import { StyledTable } from '../../entity/shared/components/styled/StyledTable'; import { ANTD_GRAY } from '../../entity/shared/constants'; -import { CLI_EXECUTOR_ID } from './utils'; +import { CLI_EXECUTOR_ID, getIngestionSourceStatus } from './utils'; import { LastStatusColumn, TypeColumn, @@ -123,7 +123,7 @@ function IngestionSourceTable({ lastExecStatus: source.executions && source.executions?.executionRequests.length > 0 && - source.executions?.executionRequests[0].result?.status, + getIngestionSourceStatus(source.executions?.executionRequests[0].result), cliIngestion: source.config?.executorId === CLI_EXECUTOR_ID, })); diff --git a/datahub-web-react/src/app/ingest/source/builder/CreateScheduleStep.tsx b/datahub-web-react/src/app/ingest/source/builder/CreateScheduleStep.tsx index 3745ee0f44dc0..9207b9d0cfcf9 100644 --- a/datahub-web-react/src/app/ingest/source/builder/CreateScheduleStep.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/CreateScheduleStep.tsx @@ -10,6 +10,7 @@ import { TimezoneSelect } from './TimezoneSelect'; import { ANTD_GRAY, REDESIGN_COLORS } from '../../../entity/shared/constants'; import { lowerFirstLetter } from '../../../shared/textUtil'; import { IngestionSourceBuilderStep } from './steps'; +import { RequiredFieldForm } from '../../../shared/form/RequiredFieldForm'; const Section = styled.div` display: flex; @@ -31,10 +32,25 @@ const CronText = styled(Typography.Paragraph)` color: ${ANTD_GRAY[7]}; `; +const CronInput = styled(Input)` + margin-bottom: 8px; + max-width: 200px; +`; + +const Schedule = styled.div` + display: flex; + align-items: center; + justify-content: start; +`; + +const AdvancedSchedule = styled.div` + margin-left: 20px; +`; + const AdvancedCheckBox = styled(Typography.Text)` margin-right: 10px; - margin-bottom: 8px; `; + const CronSuccessCheck = styled(CheckCircleOutlined)` color: ${REDESIGN_COLORS.BLUE}; margin-right: 4px; @@ -123,9 +139,9 @@ export const CreateScheduleStep = ({ state, updateState, goTo, prev }: StepProps
Configure an Ingestion Schedule
- + Run on a schedule (Recommended) @@ -141,29 +157,31 @@ export const CreateScheduleStep = ({ state, updateState, goTo, prev }: StepProps )} Schedule}> -
- Advanced - setAdvancedCronCheck(event.target.checked)} - /> -
- {advancedCronCheck ? ( - setScheduleCronInterval(e.target.value)} - /> - ) : ( - - )} + + {advancedCronCheck ? ( + setScheduleCronInterval(e.target.value)} + /> + ) : ( + + )} + + Show Advanced + setAdvancedCronCheck(event.target.checked)} + /> + + {cronAsText.error && <>Invalid cron schedule. Cron must be of UNIX form:} {!cronAsText.text && ( @@ -183,7 +201,7 @@ export const CreateScheduleStep = ({ state, updateState, goTo, prev }: StepProps Choose a timezone for the schedule. - +
@@ -191,6 +209,7 @@ export const CreateScheduleStep = ({ state, updateState, goTo, prev }: StepProps data-testid="ingestion-schedule-next-button" disabled={!interval || interval.length === 0 || cronAsText.error} onClick={onClickNext} + type="primary" > Next diff --git a/datahub-web-react/src/app/ingest/source/builder/DataPlatformCard.tsx b/datahub-web-react/src/app/ingest/source/builder/DataPlatformCard.tsx new file mode 100644 index 0000000000000..34efbb3000829 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/DataPlatformCard.tsx @@ -0,0 +1,68 @@ +import React from 'react'; +import { Button, Image } from 'antd'; +import styled from 'styled-components'; + +import { REDESIGN_COLORS } from '../../../entity/shared/constants'; + +const Container = styled(Button)` + padding: 32px; + height: 200px; + display: flex; + justify-content: center; + border-radius: 8px; + align-items: start; + flex-direction: column; + border: 1px solid #e0e0e0; + background-color: #ffffff; + &&:hover { + border: 1px solid ${REDESIGN_COLORS.BLUE}; + background-color: #ffffff; + } + white-space: unset; +`; + +const PlatformLogo = styled(Image)` + max-height: 32px; + height: 32px; + width: auto; + object-fit: contain; + background-color: transparent; +`; + +const LogoContainer = styled.div` + margin-bottom: 14px; +`; + +const Title = styled.div` + word-break: break-word; + color: #464646; + font-weight: bold; + font-size: 16px; + margin-bottom: 8px; +`; + +const Description = styled.div` + word-break: break-word; + text-align: left; + color: #7c7c7c; +`; + +type Props = { + logoUrl?: string; + logoComponent?: React.ReactNode; + name: string; + description?: string; + onClick?: () => void; +}; + +export const DataPlatformCard = ({ logoUrl, logoComponent, name, description, onClick }: Props) => { + return ( + + + {(logoUrl && ) || logoComponent} + + {name} + {description} + + ); +}; diff --git a/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx b/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx index 4ff4623b548c9..c16193b061b79 100644 --- a/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx @@ -164,7 +164,7 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev, ingestionSour - diff --git a/datahub-web-react/src/app/ingest/source/builder/IngestionDocumentationHint.tsx b/datahub-web-react/src/app/ingest/source/builder/IngestionDocumentationHint.tsx new file mode 100644 index 0000000000000..bda3d7f7424af --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/IngestionDocumentationHint.tsx @@ -0,0 +1,68 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Button, Tooltip } from 'antd'; +import { CloseOutlined } from '@ant-design/icons'; + +import { SourceConfig } from './types'; +import { ANTD_GRAY } from '../../../entity/shared/constants'; + +const Container = styled.div` + background-color: #ffffff; + border-radius: 8px; + padding: 12px 12px 16px 24px; + border: 1px solid #e0e0e0; + margin-bottom: 20px; +`; + +const Header = styled.div` + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 12px; +`; + +const Title = styled.div` + font-size: 16px; + font-weight: bold; +`; + +const Description = styled.div` + font-size: 14px; + max-width: 90%; +`; + +const StyledCloseOutlined = styled(CloseOutlined)` + color: ${ANTD_GRAY[6]}; +`; + +interface Props { + sourceConfigs: SourceConfig; + onHide: () => void; +} + +export const IngestionDocumentationHint = ({ sourceConfigs, onHide }: Props) => { + const { displayName, docsUrl } = sourceConfigs; + return ( + +
+ Let's get connected! 🎉 + +
+ +
+ To import from {displayName}, we'll need some more information to connect to your instance. +
+
+ Check out the{' '} + + {displayName} Guide + {' '} + to understand the prerequisites, learn about available settings, and view examples to help connect + to the data source. +
+ + + ); +}; diff --git a/datahub-web-react/src/app/ingest/source/builder/IngestionSourceBuilderModal.tsx b/datahub-web-react/src/app/ingest/source/builder/IngestionSourceBuilderModal.tsx index 5a623b58af5c9..a41a8ec0f12ab 100644 --- a/datahub-web-react/src/app/ingest/source/builder/IngestionSourceBuilderModal.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/IngestionSourceBuilderModal.tsx @@ -1,8 +1,7 @@ -import { Button, Modal, Steps, Typography } from 'antd'; +import { Modal, Steps, Typography } from 'antd'; import React, { useEffect, useRef, useState } from 'react'; import styled from 'styled-components'; import { isEqual } from 'lodash'; -import { ExpandAltOutlined, ShrinkOutlined } from '@ant-design/icons'; import { SourceBuilderState, StepProps } from './types'; import { CreateScheduleStep } from './CreateScheduleStep'; import { DefineRecipeStep } from './DefineRecipeStep'; @@ -10,15 +9,18 @@ import { NameSourceStep } from './NameSourceStep'; import { SelectTemplateStep } from './SelectTemplateStep'; import sourcesJson from './sources.json'; -const ExpandButton = styled(Button)` - && { - margin-right: 32px; +const StyledModal = styled(Modal)` + && .ant-modal-content { + border-radius: 16px; + overflow: hidden; + min-width: 400px; } `; const TitleContainer = styled.div` display: flex; justify-content: space-between; + border-radius: 12px; `; const StepsContainer = styled.div` @@ -31,9 +33,9 @@ const StepsContainer = styled.div` * Mapping from the step type to the title for the step */ export enum IngestionSourceBuilderStepTitles { - SELECT_TEMPLATE = 'Choose Type', - DEFINE_RECIPE = 'Configure Recipe', - CREATE_SCHEDULE = 'Schedule Ingestion', + SELECT_TEMPLATE = 'Choose Data Source', + DEFINE_RECIPE = 'Configure Connection', + CREATE_SCHEDULE = 'Sync Schedule', NAME_SOURCE = 'Finish up', } @@ -57,6 +59,8 @@ export enum IngestionSourceBuilderStep { NAME_SOURCE = 'NAME_SOURCE', } +const modalBodyStyle = { padding: '16px 24px 16px 24px', backgroundColor: '#F6F6F6' }; + type Props = { initialState?: SourceBuilderState; visible: boolean; @@ -66,14 +70,17 @@ type Props = { export const IngestionSourceBuilderModal = ({ initialState, visible, onSubmit, onCancel }: Props) => { const isEditing = initialState !== undefined; - const titleText = isEditing ? 'Edit Ingestion Source' : 'New Ingestion Source'; + const titleText = isEditing ? 'Edit Data Source' : 'Connect Data Source'; const initialStep = isEditing ? IngestionSourceBuilderStep.DEFINE_RECIPE : IngestionSourceBuilderStep.SELECT_TEMPLATE; const [stepStack, setStepStack] = useState([initialStep]); - const [modalExpanded, setModalExpanded] = useState(false); - const [ingestionBuilderState, setIngestionBuilderState] = useState({}); + const [ingestionBuilderState, setIngestionBuilderState] = useState({ + schedule: { + interval: '0 0 * * *', + }, + }); const ingestionSources = JSON.parse(JSON.stringify(sourcesJson)); // TODO: replace with call to server once we have access to dynamic list of sources @@ -122,28 +129,28 @@ export const IngestionSourceBuilderModal = ({ initialState, visible, onSubmit, o const StepComponent: React.FC = IngestionSourceBuilderStepComponent[currentStep]; return ( - {titleText} - setModalExpanded(!modalExpanded)}> - {(modalExpanded && ) || } - } style={{ top: 40 }} + bodyStyle={modalBodyStyle} visible={visible} onCancel={onCancel} > - - - {Object.keys(IngestionSourceBuilderStep).map((item) => ( - - ))} - - + {currentStepIndex > 0 ? ( + + + {Object.keys(IngestionSourceBuilderStep).map((item) => ( + + ))} + + + ) : null} - + ); }; diff --git a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx index 6f115610c7d82..898fbd6a6d926 100644 --- a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx @@ -1,7 +1,8 @@ -import { Button, Checkbox, Collapse, Form, Input, Typography } from 'antd'; +import { Button, Checkbox, Collapse, Form, Input, Tooltip, Typography } from 'antd'; import React from 'react'; import styled from 'styled-components'; import { SourceBuilderState, StepProps, StringMapEntryInput } from './types'; +import { RequiredFieldForm } from '../../../shared/form/RequiredFieldForm'; const ControlsContainer = styled.div` display: flex; @@ -156,7 +157,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps) return ( <> -
+ - Give this ingestion source a name. + Give this data source a name Advanced} key="1"> - Executor Id}> + Executor ID}> - Provide the executor id to route execution requests to. The built-in DataHub executor id - is 'default'. Do not change this unless you have configured a custom executor. + Provide the ID of the executor that should execute this ingestion recipe. This ID is + used to route execution requests of the recipe to the executor of the same ID. The + built-in DataHub executor ID is 'default'. Do not change this unless you have + configured a remote or custom executor. - +
@@ -261,13 +264,15 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps) > Save - + + +
diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx index 880420386fa67..2d0bfe340c506 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx @@ -10,6 +10,7 @@ import { SourceBuilderState, SourceConfig } from './types'; import { CSV, LOOKER, LOOK_ML } from './constants'; import { LookerWarning } from './LookerWarning'; import { CSVInfo } from './CSVInfo'; +import { IngestionDocumentationHint } from './IngestionDocumentationHint'; export const ControlsContainer = styled.div` display: flex; @@ -66,6 +67,7 @@ function RecipeBuilder(props: Props) { const { state, isEditing, displayRecipe, sourceConfigs, setStagedRecipe, onClickNext, goToPrevious } = props; const { type } = state; const [isViewingForm, setIsViewingForm] = useState(true); + const [hideDocsHint, setHideDocsHint] = useState(false); function switchViews(isFormView: boolean) { try { @@ -81,12 +83,14 @@ function RecipeBuilder(props: Props) { return (
+ {!hideDocsHint && isViewingForm && sourceConfigs ? ( + setHideDocsHint(true)} sourceConfigs={sourceConfigs} /> + ) : null} {(type === LOOKER || type === LOOK_ML) && } {type === CSV && } - - {sourceConfigs?.displayName} Recipe + {sourceConfigs?.displayName} Details Previous - diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx index bdee01d6498ee..4199658568b9a 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx @@ -1,9 +1,11 @@ -import { Button, Collapse, Form, message, Tooltip, Typography } from 'antd'; import React, { Fragment } from 'react'; + +import { Button, Collapse, Form, message, Tooltip, Typography } from 'antd'; import { get } from 'lodash'; import YAML from 'yamljs'; import { ApiOutlined, FilterOutlined, QuestionCircleOutlined, SettingOutlined } from '@ant-design/icons'; import styled from 'styled-components/macro'; + import { jsonToYaml } from '../../utils'; import { CONNECTORS_WITH_TEST_CONNECTION, RecipeSections, RECIPE_FIELDS } from './constants'; import FormField from './FormField'; @@ -11,6 +13,7 @@ import TestConnectionButton from './TestConnection/TestConnectionButton'; import { useListSecretsQuery } from '../../../../../graphql/ingestion.generated'; import { RecipeField, setFieldValueOnRecipe } from './common'; import { SourceBuilderState, SourceConfig } from '../types'; +import { RequiredFieldForm } from '../../../../shared/form/RequiredFieldForm'; export const ControlsContainer = styled.div` display: flex; @@ -140,7 +143,7 @@ function RecipeForm(props: Props) { } return ( -
} - text="Advanced" + text="Settings" sectionTooltip={advancedSectionTooltip} /> } @@ -230,9 +233,11 @@ function RecipeForm(props: Props) { - + - + ); } diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/__tests__/utils.test.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/__tests__/utils.test.ts index 160074d94abac..a2c4de8bbd3a3 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/__tests__/utils.test.ts +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/__tests__/utils.test.ts @@ -12,9 +12,7 @@ describe('validateURL function', () => { const validator = validateURL('test url'); await expect(validator.validator(null, 'http://example')).rejects.toThrowError('A valid test url is required.'); await expect(validator.validator(null, 'example')).rejects.toThrowError('A valid test url is required.'); - await expect(validator.validator(null, 'http://example')).rejects.toThrowError( - 'A valid test url is required.', - ); + await expect(validator.validator(null, 'http://example')).rejects.toThrowError('A valid test url is required.'); }); it('should resolve if the value is empty', async () => { diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx index 43d899301c2fc..cbaf2f4d87991 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx @@ -276,7 +276,7 @@ export const INCLUDE_LINEAGE: RecipeField = { export const INCLUDE_TABLE_LINEAGE: RecipeField = { name: 'include_table_lineage', label: 'Include Table Lineage', - tooltip: 'Extract Tabel-Level lineage metadata. Enabling this may increase the duration of the extraction process.', + tooltip: 'Extract Tabel-Level lineage metadata. Enabling this may increase the duration of the sync.', type: FieldType.BOOLEAN, fieldPath: 'source.config.include_table_lineage', rules: null, @@ -286,8 +286,7 @@ const isProfilingEnabledFieldPath = 'source.config.profiling.enabled'; export const TABLE_PROFILING_ENABLED: RecipeField = { name: 'profiling.enabled', label: 'Enable Table Profiling', - tooltip: - 'Generate Data Profiles for extracted Tables. Enabling this may increase the duration of the extraction process.', + tooltip: 'Generate Data Profiles for extracted Tables. Enabling this may increase the duration of the sync.', type: FieldType.BOOLEAN, fieldPath: isProfilingEnabledFieldPath, rules: null, @@ -298,7 +297,7 @@ export const COLUMN_PROFILING_ENABLED: RecipeField = { name: 'column_profiling.enabled', label: 'Enable Column Profiling', tooltip: - 'Generate Data Profiles for the Columns in extracted Tables. Enabling this may increase the duration of the extraction process.', + 'Generate Data Profiles for the Columns in extracted Tables. Enabling this may increase the duration of the sync.', type: FieldType.BOOLEAN, fieldPath: isTableProfilingOnlyFieldPath, rules: null, @@ -466,7 +465,7 @@ export const START_TIME: RecipeField = { name: 'start_time', label: 'Start Time', tooltip: - 'Earliest date used when processing audit logs for lineage, usage, and more. Default: Last full day in UTC or last time DataHub ingested usage (if stateful ingestion is enabled). Tip: Set this to an older date (e.g. 1 month ago) to bootstrap your first ingestion run, and then reduce for subsequent runs. Changing this may increase the duration of the extraction process.', + 'Earliest date used when processing audit logs for lineage, usage, and more. Default: Last full day in UTC or last time DataHub ingested usage (if stateful ingestion is enabled). Tip: Set this to an older date (e.g. 1 month ago) to bootstrap your first ingestion run, and then reduce for subsequent runs. Changing this may increase the duration of the sync.', placeholder: 'Select date and time', type: FieldType.DATE, fieldPath: startTimeFieldPath, diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/okta.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/okta.ts index 6efee3769f908..ccb16f41e9a6c 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/okta.ts +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/okta.ts @@ -150,4 +150,3 @@ export const SKIP_USERS_WITHOUT_GROUP: RecipeField = { fieldPath: 'source.config.skip_users_without_a_group', rules: null, }; - diff --git a/datahub-web-react/src/app/ingest/source/builder/SelectTemplateStep.tsx b/datahub-web-react/src/app/ingest/source/builder/SelectTemplateStep.tsx index 6b771d459c4ef..3998915e07a2c 100644 --- a/datahub-web-react/src/app/ingest/source/builder/SelectTemplateStep.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/SelectTemplateStep.tsx @@ -1,39 +1,60 @@ +import React, { useState } from 'react'; + import { Button, Input } from 'antd'; import { FormOutlined, SearchOutlined } from '@ant-design/icons'; -import React, { useState } from 'react'; import styled from 'styled-components'; -import { LogoCountCard } from '../../../shared/LogoCountCard'; import { SourceConfig, SourceBuilderState, StepProps } from './types'; import { IngestionSourceBuilderStep } from './steps'; import useGetSourceLogoUrl from './useGetSourceLogoUrl'; import { CUSTOM } from './constants'; import { ANTD_GRAY } from '../../../entity/shared/constants'; +import { DataPlatformCard } from './DataPlatformCard'; -const Section = styled.div` +const Container = styled.div` + max-height: 82vh; display: flex; flex-direction: column; - padding-bottom: 12px; `; -const PlatformListContainer = styled.div` +const Section = styled.div` display: flex; - justify-content: left; - align-items: center; - flex-wrap: wrap; + flex-direction: column; + padding-bottom: 12px; + overflow: hidden; `; const CancelButton = styled(Button)` - && { - margin-left: 12px; - } + max-width: 120px; +`; + +const SearchBarContainer = styled.div` + display: flex; + justify-content: end; + width: auto; + padding-right: 12px; `; const StyledSearchBar = styled(Input)` background-color: white; - border-radius: 70px; + border-radius: 8px; box-shadow: 0px 0px 30px 0px rgb(239 239 239); - width: 45%; - margin: 0 0 15px 12px; + border: 1px solid #e0e0e0; + margin: 0 0 15px 0px; + max-width: 300px; + font-size: 16px; +`; + +const StyledSearchOutlined = styled(SearchOutlined)` + color: #a9adbd; +`; + +const PlatformListContainer = styled.div` + display: grid; + grid-template-columns: repeat(auto-fill, minmax(min(100%, 31%), 1fr)); + gap: 10px; + height: 100%; + overflow-y: auto; + padding-right: 12px; `; interface SourceOptionProps { @@ -42,7 +63,7 @@ interface SourceOptionProps { } function SourceOption({ source, onClick }: SourceOptionProps) { - const { name, displayName } = source; + const { name, displayName, description } = source; const logoUrl = useGetSourceLogoUrl(name); let logoComponent; @@ -50,7 +71,15 @@ function SourceOption({ source, onClick }: SourceOptionProps) { logoComponent = ; } - return ; + return ( + + ); } /** @@ -76,22 +105,24 @@ export const SelectTemplateStep = ({ state, updateState, goTo, cancel, ingestion ); return ( - <> +
- setSearchFilter(e.target.value)} - allowClear - prefix={} - /> - + + setSearchFilter(e.target.value)} + allowClear + prefix={} + /> + + {filteredSources.map((source) => ( onSelectTemplate(source.name)} /> ))}
Cancel - + ); }; diff --git a/datahub-web-react/src/app/ingest/source/builder/TimezoneSelect.tsx b/datahub-web-react/src/app/ingest/source/builder/TimezoneSelect.tsx index d9f3df1fc9929..21731b69cf46b 100644 --- a/datahub-web-react/src/app/ingest/source/builder/TimezoneSelect.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/TimezoneSelect.tsx @@ -1,21 +1,28 @@ import { Select } from 'antd'; import React from 'react'; import moment from 'moment-timezone'; +import styled from 'styled-components'; + +const StyledSelect = styled(Select)` + max-width: 300px; +`; type Props = { value: string; - onChange: (newTimezone: string) => void; + onChange: (newTimezone: any) => void; }; export const TimezoneSelect = ({ value, onChange }: Props) => { const timezones = moment.tz.names(); return ( <> - + ); }; diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index d4faf82a20605..c35a7a033a8ab 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -3,55 +3,63 @@ "urn": "urn:li:dataPlatform:bigquery", "name": "bigquery", "displayName": "BigQuery", - "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/bigquery/", + "description": "Import Projects, Datasets, Tables, Views, lineage, queries, and statistics from BigQuery.", + "docsUrl": "https://datahubproject.io/docs/quick-ingestion-guides/bigquery/overview", "recipe": "source:\n type: bigquery\n config:\n include_table_lineage: true\n include_usage_statistics: true\n include_tables: true\n include_views: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true" }, { "urn": "urn:li:dataPlatform:redshift", "name": "redshift", "displayName": "Redshift", - "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/redshift/", + "description": "Import Tables, Views, Databases, Schemas, lineage, queries, and statistics from Redshift.", + "docsUrl": "https://datahubproject.io/docs/quick-ingestion-guides/redshift/overview", "recipe": "source: \n type: redshift\n config:\n # Coordinates\n host_port: # Your Redshift host and post, e.g. example.something.us-west-2.redshift.amazonaws.com:5439\n database: # Your Redshift database, e.g. SampleDatabase\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: null # Your Redshift username, e.g. admin\n\n table_lineage_mode: stl_scan_based\n include_table_lineage: true\n include_tables: true\n include_views: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true" }, { "urn": "urn:li:dataPlatform:snowflake", "name": "snowflake", "displayName": "Snowflake", - "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/snowflake/", + "description": "Import Tables, Views, Databases, Schemas, lineage, queries, and statistics from Snowflake.", + "docsUrl": "https://datahubproject.io/docs/quick-ingestion-guides/snowflake/overview", "recipe": "source: \n type: snowflake\n config:\n account_id: null\n include_table_lineage: true\n include_view_lineage: true\n include_tables: true\n include_views: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true" }, { - "urn": "urn:li:dataPlatform:kafka", - "name": "kafka", - "displayName": "Kafka", - "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/kafka/", - "recipe": "source:\n type: kafka\n config:\n connection:\n consumer_config:\n security.protocol: \"PLAINTEXT\"\n stateful_ingestion:\n enabled: false" + "urn": "urn:li:dataPlatform:unity-catalog", + "name": "unity-catalog", + "displayName": "Databricks", + "description": "Import Metastores, Schemas, Tables, lineage, queries, and statistics from Databricks Unity Catalog.", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/databricks/#module-unity-catalog", + "recipe": "source:\n type: unity-catalog\n config:\n # Coordinates\n workspace_url: null\n include_table_lineage: true\n include_column_lineage: false\n stateful_ingestion:\n enabled: true" }, { "urn": "urn:li:dataPlatform:looker", "name": "looker", "displayName": "Looker", - "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/looker/", - "recipe": "source:\n type: looker\n config:\n # Coordinates\n base_url: # Your Looker instance URL, e.g. https://company.looker.com:19999\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n client_id: null # Your Looker client id, e.g. admin\n stateful_ingestion:\n enabled: true" + "description": "Import Models, Explores, Views, Looks, Dashboards, and lineage from Looker.", + "docsUrl": "https://datahubproject.io/docs/quick-ingestion-guides/looker/overview#looker", + "recipe": "source:\n type: looker\n config:\n # Coosrdinates\n base_url: # Your Looker instance URL, e.g. https://company.looker.com:19999\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n client_id: null # Your Looker client id, e.g. admin\n stateful_ingestion:\n enabled: true" }, { "urn": "urn:li:dataPlatform:lookml", "name": "lookml", "displayName": "LookML", - "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/looker/#module-lookml", + "description": "Import Models, Explores, Views, Looks, Dashboards, and lineage from LookML files.", + "docsUrl": "https://datahubproject.io/docs/quick-ingestion-guides/looker/overview#lookml", "recipe": "source:\n type: lookml\n config:\n parse_table_names_from_sql: true\n stateful_ingestion:\n enabled: true" }, { "urn": "urn:li:dataPlatform:tableau", "name": "tableau", "displayName": "Tableau", - "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/tableau/", + "description": "Import Data Sources, Workbooks, Worksheets, Tags, Dashboards, and lineage from Tableau.", + "docsUrl": "https://datahubproject.io/docs/quick-ingestion-guides/tableau/overview", "recipe": "source:\n type: tableau\n config:\n # Coordinates\n connect_uri: null\n stateful_ingestion:\n enabled: true" }, { "urn": "urn:li:dataPlatform:powerbi", "name": "powerbi", "displayName": "PowerBI", + "description": "Import Dashboards, Tiles, Datasets, and lineage from PowerBI.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/powerbi/", "recipe": "source:\n type: \"powerbi\"\n config:\n # Your Power BI tenant identifier\n tenant_id: null\n # Your Power BI client id\n client_id: null\n # Your Power BI client secret\n client_secret: null\n stateful_ingestion:\n enabled: true" }, @@ -59,6 +67,7 @@ "urn": "urn:li:dataPlatform:dbt", "name": "dbt-cloud", "displayName": "dbt Cloud", + "description": "Import Sources, Seeds, Models, Snapshots, Tests, and lineage from dbt cloud.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/dbt/#module-dbt-cloud", "recipe": "source:\n type: dbt-cloud\n config:\n account_id: null\n project_id: null\n job_id: null\n target_platform: null\n stateful_ingestion:\n enabled: true" }, @@ -66,6 +75,7 @@ "urn": "urn:li:dataPlatform:mysql", "name": "mysql", "displayName": "MySQL", + "description": "Import Tables, Views, Databases, Schemas, and statistics from MySQL.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mysql/", "recipe": "source: \n type: mysql\n config: \n # Coordinates\n host_port: # Your MySQL host and post, e.g. mysql:3306\n database: # Your MySQL database name, e.g. datahub\n \n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: null # Your MySQL username, e.g. admin\n\n # Options\n include_tables: true\n include_views: true\n\n # Profiling\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true" }, @@ -73,13 +83,23 @@ "urn": "urn:li:dataPlatform:postgres", "name": "postgres", "displayName": "Postgres", + "description": "Import Tables, Views, Databases, Schemas, and statistics from Postgres.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/postgres/", "recipe": "source: \n type: postgres\n config:\n # Coordinates\n host_port: # Your Postgres host and port, e.g. postgres:5432\n database: # Your Postgres Database, e.g. sample_db\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: null # Your Postgres username, e.g. admin\n\n # Options\n include_tables: true\n include_views: true\n\n # Profiling\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true" }, + { + "urn": "urn:li:dataPlatform:kafka", + "name": "kafka", + "displayName": "Kafka", + "description": "Import streaming topics from Kafka.", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/kafka/", + "recipe": "source:\n type: kafka\n config:\n connection:\n consumer_config:\n security.protocol: \"PLAINTEXT\"\n stateful_ingestion:\n enabled: false" + }, { "urn": "urn:li:dataPlatform:hive", "name": "hive", "displayName": "Hive", + "description": "Import Tables, Views, Databases, Schemas, and statistics from Hive.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/hive/", "recipe": "source: \n type: hive\n config:\n # Coordinates\n host_port: # Your Hive host and port, e.g. hive:10000\n database: # Your Hive database name, e.g. SampleDatabase (Optional, if not specified, ingests from all databases)\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: null # Your Hive username, e.g. admin\n stateful_ingestion:\n enabled: true" }, @@ -87,6 +107,7 @@ "urn": "urn:li:dataPlatform:presto", "name": "presto", "displayName": "Presto", + "description": "Import Tables, Databases, Schemas, and statistics from Presto.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/presto/", "recipe": "source:\n type: presto\n config:\n # Coordinates\n host_port: null\n # The name of the catalog from getting the usage\n database: null\n # Credentials\n username: null\n include_views: true\n include_tables: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true" }, @@ -94,13 +115,23 @@ "urn": "urn:li:dataPlatform:trino", "name": "trino", "displayName": "Trino", + "description": "Import Tables, Databases, Schemas, and statistics from Trino.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/trino/", "recipe": "source:\n type: trino\n config:\n # Coordinates\n host_port: null\n # The name of the catalog from getting the usage\n database: null\n # Credentials\n username: null\n include_views: true\n include_tables: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true" }, + { + "urn": "urn:li:dataPlatform:glue", + "name": "glue", + "displayName": "Glue", + "description": "Import Tables, Databases, Jobs, statistics, and lineage to S3 from AWS Glue.", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/glue/", + "recipe": "source:\n type: glue\n config:\n # AWS credentials. \n aws_region: # The region for your AWS Glue instance. \n # Add secret in Secrets Tab with relevant names for each variable\n # The access key for your AWS account.\n aws_access_key_id: \"${AWS_ACCESS_KEY_ID}\"\n # The secret key for your AWS account.\n aws_secret_access_key: \"${AWS_SECRET_KEY}\"\n aws_session_token: # The session key for your AWS account. This is only needed when you are using temporary credentials.\n # aws_role: # (Optional) The role to assume (Role chaining supported by using a sorted list).\n\n # Allow / Deny specific databases & tables\n # database_pattern:\n # allow:\n # - \"flights-database\"\n # table_pattern:\n # allow:\n # - \"avro\"" + }, { "urn": "urn:li:dataPlatform:mssql", "name": "mssql", "displayName": "Microsoft SQL Server", + "description": "Import Tables, Views, Databases, Schemas, and statistics from SQL Server.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mssql/", "recipe": "source:\n type: mssql\n config:\n # Coordinates\n host_port: null\n # The name\n database: null\n # Credentials\n username: null\n include_views: true\n include_tables: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true" }, @@ -108,20 +139,15 @@ "urn": "urn:li:dataPlatform:mariadb", "name": "mariadb", "displayName": "MariaDB", + "description": "Import Tables, Views, Databases, Schemas, and statistics from MariaDB.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mariadb/", "recipe": "source:\n type: mariadb\n config:\n # Coordinates\n host_port: null\n # The name\n database: null\n # Credentials\n username: null\n include_views: true\n include_tables: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true" }, - { - "urn": "urn:li:dataPlatform:unity-catalog", - "name": "unity-catalog", - "displayName": "Databricks Unity Catalog", - "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/databricks/#module-unity-catalog", - "recipe": "source:\n type: unity-catalog\n config:\n # Coordinates\n workspace_url: null\n include_table_lineage: true\n include_column_lineage: false\n stateful_ingestion:\n enabled: true" - }, { "urn": "urn:li:dataPlatform:mongodb", "name": "mongodb", "displayName": "MongoDB", + "description": "Import Databases and Collections from MongoDB.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mongodb/", "recipe": "source:\n type: mongodb\n config:\n # Coordinates\n connect_uri: # Your MongoDB connect URI, e.g. \"mongodb://localhost\"\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: \"${MONGO_USERNAME}\" # Your MongoDB username, e.g. admin\n password: \"${MONGO_PASSWORD}\" # Your MongoDB password, e.g. password_01\n\n # Options (recommended)\n enableSchemaInference: True\n useRandomSampling: True\n maxSchemaSize: 300" }, @@ -129,20 +155,15 @@ "urn": "urn:li:dataPlatform:dynamodb", "name": "dynamodb", "displayName": "DynamoDB", + "description": "Import Tables from DynamoDB.", "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/", "recipe": "source:\n type: dynamodb\n config:\n platform_instance: \"AWS_ACCOUNT_ID\"\n aws_access_key_id : '${AWS_ACCESS_KEY_ID}'\n aws_secret_access_key : '${AWS_SECRET_ACCESS_KEY}'\n # If there are items that have most representative fields of the table, users could use the\n # `include_table_item` option to provide a list of primary keys of the table in dynamodb format.\n # For each `region.table`, the list of primary keys can be at most 100.\n # We include these items in addition to the first 100 items in the table when we scan it.\n # include_table_item:\n # region.table_name:\n # [\n # {\n # 'partition_key_name': { 'attribute_type': 'attribute_value' },\n # 'sort_key_name': { 'attribute_type': 'attribute_value' },\n # },\n # ]" }, - { - "urn": "urn:li:dataPlatform:glue", - "name": "glue", - "displayName": "Glue", - "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/glue/", - "recipe": "source:\n type: glue\n config:\n # AWS credentials. \n aws_region: # The region for your AWS Glue instance. \n # Add secret in Secrets Tab with relevant names for each variable\n # The access key for your AWS account.\n aws_access_key_id: \"${AWS_ACCESS_KEY_ID}\"\n # The secret key for your AWS account.\n aws_secret_access_key: \"${AWS_SECRET_KEY}\"\n aws_session_token: # The session key for your AWS account. This is only needed when you are using temporary credentials.\n # aws_role: # (Optional) The role to assume (Role chaining supported by using a sorted list).\n\n # Allow / Deny specific databases & tables\n # database_pattern:\n # allow:\n # - \"flights-database\"\n # table_pattern:\n # allow:\n # - \"avro\"" - }, { "urn": "urn:li:dataPlatform:oracle", "name": "oracle", "displayName": "Oracle", + "description": "Import Databases, Schemas, Tables, Views, statistics, and lineage from Oracle.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/oracle/", "recipe": "source: \n type: oracle\n config:\n # Coordinates\n host_port: # Your Oracle host and port, e.g. oracle:5432\n database: # Your Oracle database name, e.g. sample_db\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: \"${ORACLE_USERNAME}\" # Your Oracle username, e.g. admin\n password: \"${ORACLE_PASSWORD}\" # Your Oracle password, e.g. password_01\n\n # Optional service name\n # service_name: # Your service name, e.g. svc # omit database if using this option" }, @@ -150,6 +171,7 @@ "urn": "urn:li:dataPlatform:superset", "name": "superset", "displayName": "Superset", + "description": "Import Charts and Dashboards from Superset", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/superset/", "recipe": "source:\n type: superset\n config:\n # Coordinates\n connect_uri: http://localhost:8088\n\n # Credentials\n username: user\n password: pass\n provider: ldap" }, @@ -157,6 +179,7 @@ "urn": "urn:li:dataPlatform:athena", "name": "athena", "displayName": "Athena", + "description": "Import Schemas, Tables, Views, and lineage to S3 from Athena.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/athena/", "recipe": "source:\n type: athena\n config:\n # Coordinates\n aws_region: my_aws_region\n work_group: primary\n\n # Options\n s3_staging_dir: \"s3://my_staging_athena_results_bucket/results/\"" }, @@ -164,6 +187,7 @@ "urn": "urn:li:dataPlatform:clickhouse", "name": "clickhouse", "displayName": "ClickHouse", + "description": "Import Tables, Views, Materialized Views, Dictionaries, statistics, queries, and lineage from ClickHouse.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/clickhouse/", "recipe": "source:\n type: clickhouse\n config:\n # Coordinates\n host_port: localhost:9000\n\n # Credentials\n username: user\n password: pass\n\n # Options\n platform_instance: DatabaseNameToBeIngested\n\n include_views: true # whether to include views, defaults to True\n include_tables: true # whether to include views, defaults to True\n\nsink:\n # sink configs\n\n#---------------------------------------------------------------------------\n# For the HTTP interface:\n#---------------------------------------------------------------------------\nsource:\n type: clickhouse\n config:\n host_port: localhost:8443\n protocol: https\n\n#---------------------------------------------------------------------------\n# For the Native interface:\n#---------------------------------------------------------------------------\n\nsource:\n type: clickhouse\n config:\n host_port: localhost:9440\n scheme: clickhouse+native\n secure: True" }, @@ -171,13 +195,23 @@ "urn": "urn:li:dataPlatform:druid", "name": "druid", "displayName": "Druid", + "description": "Import Databases, Schemas, Tables, statistics, and lineage from Druid.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/druid/", "recipe": "source:\n type: druid\n config:\n # Coordinates\n host_port: \"localhost:8082\"\n\n # Credentials\n username: admin\n password: password" }, + { + "urn": "urn:li:dataPlatform:mode", + "name": "mode", + "displayName": "Mode", + "description": "Import Reports, Charts, and lineage from Mode.", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mode/", + "recipe": "source:\n type: mode\n config:\n # Coordinates\n connect_uri: http://app.mode.com\n\n # Credentials\n token: token\n password: pass\n\n # Options\n workspace: \"datahub\"\n default_schema: \"public\"\n owner_username_instead_of_email: False\n api_options:\n retry_backoff_multiplier: 2\n max_retry_interval: 10\n max_attempts: 5" + }, { "urn": "urn:li:dataPlatform:metabase", "name": "metabase", "displayName": "Metabase", + "description": "Import Collections, Dashboards, and Charts from Metabase.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/metabase/", "recipe": "source:\n type: metabase\n config:\n # Coordinates\n connect_uri:\n\n # Credentials\n username: root\n password: example" }, @@ -185,20 +219,15 @@ "urn": "urn:li:dataPlatform:mlflow", "name": "mlflow", "displayName": "MLflow", + "description": "Import Registered Models, Model Versions, and Model Stages from MLflow.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mlflow/", "recipe": "source:\n type: mlflow\n config:\n tracking_uri: tracking_uri" }, - { - "urn": "urn:li:dataPlatform:mode", - "name": "mode", - "displayName": "Mode", - "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mode/", - "recipe": "source:\n type: mode\n config:\n # Coordinates\n connect_uri: http://app.mode.com\n\n # Credentials\n token: token\n password: pass\n\n # Options\n workspace: \"datahub\"\n default_schema: \"public\"\n owner_username_instead_of_email: False\n api_options:\n retry_backoff_multiplier: 2\n max_retry_interval: 10\n max_attempts: 5" - }, { "urn": "urn:li:dataPlatform:azure-ad", "name": "azure-ad", "displayName": "Azure AD", + "description": "Import Users and Groups from Azure Active Directory.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/azure-ad/", "recipe": "source:\n type: azure-ad\n config:\n client_id: # Your Azure Client ID, e.g. \"00000000-0000-0000-0000-000000000000\"\n tenant_id: # Your Azure Tenant ID, e.g. \"00000000-0000-0000-0000-000000000000\"\n # Add secret in Secrets Tab with this name\n client_secret: \n redirect: # Your Redirect URL, e.g. \"https://login.microsoftonline.com/common/oauth2/nativeclient\"\n authority: # Your Authority URL, e.g. \"https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000\"\n token_url: # Your Token URL, e.g. \"https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/token\"\n graph_url: # The Graph URL, e.g. \"https://graph.microsoft.com/v1.0\"\n \n # Optional flags to ingest users, groups, or both\n ingest_users: True\n ingest_groups: True\n \n # Optional Allow / Deny extraction of particular Groups\n # groups_pattern:\n # allow:\n # - \".*\"\n\n # Optional Allow / Deny extraction of particular Users.\n # users_pattern:\n # allow:\n # - \".*\"" }, @@ -206,6 +235,7 @@ "urn": "urn:li:dataPlatform:okta", "name": "okta", "displayName": "Okta", + "description": "Import Users and Groups from Okta.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/okta/", "recipe": "source:\n type: okta\n config:\n # Coordinates\n okta_domain: # Your Okta Domain, e.g. \"dev-35531955.okta.com\"\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n okta_api_token: # Your Okta API Token, e.g. \"11be4R_M2MzDqXawbTHfKGpKee0kuEOfX1RCQSRx99\"\n\n # Optional flags to ingest users, groups, or both\n ingest_users: True\n ingest_groups: True\n\n # Optional: Customize the mapping to DataHub Username from an attribute appearing in the Okta User\n # profile. Reference: https://developer.okta.com/docs/reference/api/users/\n # okta_profile_to_username_attr: str = \"login\"\n # okta_profile_to_username_regex: str = \"([^@]+)\"\n \n # Optional: Customize the mapping to DataHub Group from an attribute appearing in the Okta Group\n # profile. Reference: https://developer.okta.com/docs/reference/api/groups/\n # okta_profile_to_group_name_attr: str = \"name\"\n # okta_profile_to_group_name_regex: str = \"(.*)\"\n \n # Optional: Include deprovisioned or suspended Okta users in the ingestion.\n # include_deprovisioned_users = False\n # include_suspended_users = False" }, @@ -213,6 +243,7 @@ "urn": "urn:li:dataPlatform:vertica", "name": "vertica", "displayName": "Vertica", + "description": "Import Databases, Schemas, Tables, Views, Projections, statistics, and lineage from Vertica.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/vertica/", "recipe": "source:\n type: vertica\n config:\n # Coordinates\n host_port: localhost:5433\n # The name of the vertica database\n database: Database_Name\n # Credentials\n username: Vertica_User\n password: Vertica_Password\n\n include_tables: true\n include_views: true\n include_projections: true\n include_models: true\n include_view_lineage: true\n include_projection_lineage: true\n profiling:\n enabled: false\n stateful_ingestion:\n enabled: true " }, @@ -220,42 +251,48 @@ "urn": "urn:li:dataPlatform:fivetran", "name": "fivetran", "displayName": "Fivetran", + "description": "Import Connectors, Destinations, Sync Histor, Users, and lineage from FiveTran.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/fivetran/", "recipe": "source:\n type: fivetran\n config:\n # Fivetran log connector destination server configurations\n fivetran_log_config:\n destination_platform: snowflake\n snowflake_destination_config:\n # Coordinates\n account_id: snowflake_account_id\n warehouse: warehouse_name\n database: snowflake_db\n log_schema: fivetran_log_schema\n\n # Credentials\n username: ${SNOWFLAKE_USER}\n password: ${SNOWFLAKE_PASS}\n role: snowflake_role\n\n # Optional - filter for certain connector names instead of ingesting everything.\n # connector_patterns:\n # allow:\n # - connector_name\n\n # Optional -- This mapping is optional and only required to configure platform-instance for source\n # A mapping of Fivetran connector id to data platform instance\n # sources_to_platform_instance:\n # calendar_elected:\n # platform_instance: cloud_postgres_instance\n # env: DEV\n\n # Optional -- This mapping is optional and only required to configure platform-instance for destination.\n # A mapping of Fivetran destination id to data platform instance\n # destination_to_platform_instance:\n # calendar_elected:\n # platform_instance: cloud_postgres_instance\n # env: DEV" }, { - "urn": "urn:li:dataPlatform:csv-enricher", - "name": "csv-enricher", - "displayName": "CSV", - "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/csv'", - "recipe": "source: \n type: csv-enricher \n config: \n # URL of your csv file to ingest \n filename: \n array_delimiter: '|' \n delimiter: ',' \n write_semantics: PATCH" - }, - { - "urn": "urn:li:dataPlatform:custom", - "name": "custom", - "displayName": "Other", - "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/", - "recipe": "source:\n type: \n config:\n # Source-type specifics config\n " + "urn": "urn:li:dataPlatform:sigma", + "name": "sigma", + "displayName": "Sigma", + "description": "Import Workspaces, Workbooks, Pages, Elements, and lineage from Sigma Computing.", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/sigma/", + "recipe": "source:\n type: sigma\n config:\n # Coordinates\n api_url: https://aws-api.sigmacomputing.com/v2\n # Coordinates\n client_id: CLIENT_ID\n client_secret: CLIENT_SECRET\n\n # Optional - filter for certain workspace names instead of ingesting everything.\n # workspace_pattern:\n\n # allow:\n # - workspace_name\n ingest_owner: true" }, { "urn": "urn:li:dataPlatform:qlik-sense", "name": "qlik-sense", "displayName": "Qlik Sense", + "description": "Import Spaces, Apps, Sheets, Charts, and Datasets from Qlik Sense.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/qlik-sense/", "recipe": "source:\n type: qlik-sense\n config:\n # Coordinates\n tenant_hostname: https://xyz12xz.us.qlikcloud.com\n # Coordinates\n api_key: QLIK_API_KEY\n\n # Optional - filter for certain space names instead of ingesting everything.\n # space_pattern:\n\n # allow:\n # - space_name\n ingest_owner: true" }, - { - "urn": "urn:li:dataPlatform:sigma", - "name": "sigma", - "displayName": "Sigma", - "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/sigma/", - "recipe": "source:\n type: sigma\n config:\n # Coordinates\n api_url: https://aws-api.sigmacomputing.com/v2\n # Coordinates\n client_id: CLIENT_ID\n client_secret: CLIENT_SECRET\n\n # Optional - filter for certain workspace names instead of ingesting everything.\n # workspace_pattern:\n\n # allow:\n # - workspace_name\n ingest_owner: true" - }, { "urn": "urn:li:dataPlatform:cockroachdb", "name": "cockroachdb", "displayName": "CockroachDb", + "description": "Import Databases, Schemas, Tables, Views, statistics and lineage from CockroachDB.", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/cockroachdb/", "recipe": "source: \n type: cockroachdb\n config:\n # Coordinates\n host_port: # Your CockroachDb host and port, e.g. cockroachdb:5432\n database: # Your CockroachDb Database, e.g. sample_db\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: null # Your CockroachDb username, e.g. admin\n\n # Options\n include_tables: true\n include_views: true\n\n # Profiling\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true" + }, + { + "urn": "urn:li:dataPlatform:csv-enricher", + "name": "csv-enricher", + "displayName": "CSV", + "description": "Import metadata from a formatted CSV.", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/csv'", + "recipe": "source: \n type: csv-enricher \n config: \n # URL of your csv file to ingest \n filename: \n array_delimiter: '|' \n delimiter: ',' \n write_semantics: PATCH" + }, + { + "urn": "urn:li:dataPlatform:custom", + "name": "custom", + "displayName": "Other", + "description": "Configure a custom recipe using YAML.", + "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/", + "recipe": "source:\n type: \n config:\n # Source-type specifics config\n " } ] diff --git a/datahub-web-react/src/app/ingest/source/builder/types.ts b/datahub-web-react/src/app/ingest/source/builder/types.ts index 2df467b7beba1..e42bd0b790b2c 100644 --- a/datahub-web-react/src/app/ingest/source/builder/types.ts +++ b/datahub-web-react/src/app/ingest/source/builder/types.ts @@ -18,6 +18,7 @@ export interface SourceConfig { name: string; displayName: string; docsUrl: string; + description?: string; recipe: string; } diff --git a/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx b/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx index 0799f8af1173d..6711f0ad12b03 100644 --- a/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx +++ b/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx @@ -13,9 +13,13 @@ import { getExecutionRequestStatusDisplayText, getExecutionRequestStatusIcon, getExecutionRequestSummaryText, + getIngestionSourceStatus, + getStructuredReport, RUNNING, SUCCESS, } from '../utils'; +import { ExecutionRequestResult } from '../../../../types.generated'; +import { StructuredReport } from './reporting/StructuredReport'; const StyledTitle = styled(Typography.Title)` padding: 0px; @@ -125,26 +129,30 @@ export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => { }; const logs = (showExpandedLogs && output) || output?.split('\n').slice(0, 5).join('\n'); - const result = data?.executionRequest?.result?.status; + const result = data?.executionRequest?.result as Partial; + const status = getIngestionSourceStatus(result); useEffect(() => { const interval = setInterval(() => { - if (result === RUNNING) refetch(); + if (status === RUNNING) refetch(); }, 2000); return () => clearInterval(interval); }); - const ResultIcon = result && getExecutionRequestStatusIcon(result); - const resultColor = result && getExecutionRequestStatusDisplayColor(result); - const resultText = result && ( + const ResultIcon = status && getExecutionRequestStatusIcon(status); + const resultColor = status && getExecutionRequestStatusDisplayColor(status); + const resultText = status && ( {ResultIcon && } - {getExecutionRequestStatusDisplayText(result)} + {getExecutionRequestStatusDisplayText(status)} ); + + const structuredReport = result && getStructuredReport(result); + const resultSummaryText = - (result && {getExecutionRequestSummaryText(result)}) || + (status && {getExecutionRequestSummaryText(status)}) || undefined; const recipeJson = data?.executionRequest?.input.arguments?.find((arg) => arg.key === 'recipe')?.value; @@ -167,21 +175,22 @@ export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => { bodyStyle={modalBodyStyle} title={ - Ingestion Run Details + Sync Details } visible={visible} onCancel={onClose} > - {!data && loading && } - {error && message.error('Failed to load execution details :(')} + {!data && loading && } + {error && message.error('Failed to load sync details :(')}
Status {resultText} {resultSummaryText} + {structuredReport ? : null} - {result === SUCCESS && ( + {status === SUCCESS && ( {data?.executionRequest?.id && } @@ -190,7 +199,7 @@ export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => { Logs - View logs that were collected during the ingestion run. + View logs that were collected during the sync.
+ {dataPlatformInstances?.length > 0 && ( +
+ Data Platform Instances + + {dataPlatformInstances.map((value, key) => { + return ( + // eslint-disable-next-line react/no-array-index-key + + {getDisplayName(value.entity)} + + ); + })} +
+ )}
Domains diff --git a/datahub-web-react/src/app/permissions/policy/policyUtils.ts b/datahub-web-react/src/app/permissions/policy/policyUtils.ts index c7ec171bc2c29..725e39d82d62e 100644 --- a/datahub-web-react/src/app/permissions/policy/policyUtils.ts +++ b/datahub-web-react/src/app/permissions/policy/policyUtils.ts @@ -179,7 +179,6 @@ export const updateListPoliciesCache = (client, policies, pageSize) => { }, }, data: { - listPolicies: { __typename: 'ListPoliciesResult', start: 0, diff --git a/datahub-web-react/src/app/permissions/policy/usePolicy.ts b/datahub-web-react/src/app/permissions/policy/usePolicy.ts index d04ea25d20b23..faa9d1cf4ab9b 100644 --- a/datahub-web-react/src/app/permissions/policy/usePolicy.ts +++ b/datahub-web-react/src/app/permissions/policy/usePolicy.ts @@ -12,25 +12,27 @@ import { PolicyUpdateInput, ResourceFilterInput, } from '../../../types.generated'; -import { useCreatePolicyMutation, useDeletePolicyMutation, useUpdatePolicyMutation } from '../../../graphql/policy.generated'; +import { + useCreatePolicyMutation, + useDeletePolicyMutation, + useUpdatePolicyMutation, +} from '../../../graphql/policy.generated'; import analytics, { EventType } from '../../analytics'; import { DEFAULT_PAGE_SIZE, removeFromListPoliciesCache, updateListPoliciesCache } from './policyUtils'; - type PrivilegeOptionType = { type?: string; name?: Maybe; }; export function usePolicy( - policiesConfig, - focusPolicyUrn, - policiesRefetch, - setShowViewPolicyModal, - onCancelViewPolicy, - onClosePolicyBuilder -){ - + policiesConfig, + focusPolicyUrn, + policiesRefetch, + setShowViewPolicyModal, + onCancelViewPolicy, + onClosePolicyBuilder, +) { const client = useApolloClient(); // Construct privileges @@ -44,22 +46,24 @@ export function usePolicy( const [deletePolicy, { error: deletePolicyError }] = useDeletePolicyMutation(); - const toFilterInput = (filter: PolicyMatchFilter,state?:string | undefined): PolicyMatchFilterInput => { - console.log({state}) + const toFilterInput = (filter: PolicyMatchFilter, state?: string | undefined): PolicyMatchFilterInput => { + console.log({ state }); return { criteria: filter.criteria?.map((criterion): PolicyMatchCriterionInput => { return { field: criterion.field, values: criterion.values.map((criterionValue) => - criterion.field === 'TAG' && state !=='TOGGLE' ? (criterionValue as any) : criterionValue.value, - ), + criterion.field === 'TAG' && state !== 'TOGGLE' + ? (criterionValue as any) + : criterionValue.value, + ), condition: criterion.condition, }; }), }; }; - const toPolicyInput = (policy: Omit,state?:string | undefined): PolicyUpdateInput => { + const toPolicyInput = (policy: Omit, state?: string | undefined): PolicyUpdateInput => { let policyInput: PolicyUpdateInput = { type: policy.type, name: policy.name, @@ -82,7 +86,7 @@ export function usePolicy( allResources: policy.resources.allResources, }; if (policy.resources.filter) { - resourceFilter = { ...resourceFilter, filter: toFilterInput(policy.resources.filter,state) }; + resourceFilter = { ...resourceFilter, filter: toFilterInput(policy.resources.filter, state) }; } // Add the resource filters. policyInput = { @@ -97,22 +101,22 @@ export function usePolicy( let privileges: PrivilegeOptionType[] = []; if (policy?.type === PolicyType.Platform) { privileges = platformPrivileges - .filter((platformPrivilege) => policy.privileges.includes(platformPrivilege.type)) - .map((platformPrivilege) => { - return { type: platformPrivilege.type, name: platformPrivilege.displayName }; + .filter((platformPrivilege) => policy.privileges.includes(platformPrivilege.type)) + .map((platformPrivilege) => { + return { type: platformPrivilege.type, name: platformPrivilege.displayName }; }); } else { const allResourcePriviliges = resourcePrivileges.find( (resourcePrivilege) => resourcePrivilege.resourceType === 'all', - ); - privileges = + ); + privileges = allResourcePriviliges?.privileges - .filter((resourcePrivilege) => policy.privileges.includes(resourcePrivilege.type)) - .map((b) => { - return { type: b.type, name: b.displayName }; - }) || []; - } - return privileges; + .filter((resourcePrivilege) => policy.privileges.includes(resourcePrivilege.type)) + .map((b) => { + return { type: b.type, name: b.displayName }; + }) || []; + } + return privileges; }; // On Delete Policy handler @@ -121,8 +125,7 @@ export function usePolicy( title: `Delete ${policy?.name}`, content: `Are you sure you want to remove policy?`, onOk() { - deletePolicy({ variables: { urn: policy?.urn as string } }) - .then(()=>{ + deletePolicy({ variables: { urn: policy?.urn as string } }).then(() => { // There must be a focus policy urn. analytics.event({ type: EventType.DeleteEntityEvent, @@ -130,12 +133,12 @@ export function usePolicy( entityType: EntityType.DatahubPolicy, }); message.success('Successfully removed policy.'); - removeFromListPoliciesCache(client,policy?.urn, DEFAULT_PAGE_SIZE); + removeFromListPoliciesCache(client, policy?.urn, DEFAULT_PAGE_SIZE); setTimeout(() => { policiesRefetch(); }, 3000); onCancelViewPolicy(); - }) + }); }, onCancel() {}, okText: 'Yes', @@ -154,20 +157,20 @@ export function usePolicy( updatePolicy({ variables: { urn: policy?.urn as string, // There must be a focus policy urn. - input: toPolicyInput(newPolicy,'TOGGLE'), + input: toPolicyInput(newPolicy, 'TOGGLE'), }, - }).then(()=>{ - const updatePolicies= { + }).then(() => { + const updatePolicies = { ...newPolicy, __typename: 'ListPoliciesResult', - } - updateListPoliciesCache(client,updatePolicies,DEFAULT_PAGE_SIZE); + }; + updateListPoliciesCache(client, updatePolicies, DEFAULT_PAGE_SIZE); message.success(`Successfully ${newState === PolicyState.Active ? 'activated' : 'deactivated'} policy.`); setTimeout(() => { policiesRefetch(); }, 3000); - }) - + }); + setShowViewPolicyModal(false); }; @@ -175,8 +178,7 @@ export function usePolicy( const onSavePolicy = (savePolicy: Omit) => { if (focusPolicyUrn) { // If there's an URN associated with the focused policy, then we are editing an existing policy. - updatePolicy({ variables: { urn: focusPolicyUrn, input: toPolicyInput(savePolicy) } }) - .then(()=>{ + updatePolicy({ variables: { urn: focusPolicyUrn, input: toPolicyInput(savePolicy) } }).then(() => { const newPolicy = { __typename: 'ListPoliciesResult', urn: focusPolicyUrn, @@ -188,16 +190,15 @@ export function usePolicy( policyUrn: focusPolicyUrn, }); message.success('Successfully saved policy.'); - updateListPoliciesCache(client,newPolicy,DEFAULT_PAGE_SIZE); + updateListPoliciesCache(client, newPolicy, DEFAULT_PAGE_SIZE); setTimeout(() => { policiesRefetch(); }, 1000); onClosePolicyBuilder(); - }) + }); } else { // If there's no URN associated with the focused policy, then we are creating. - createPolicy({ variables: { input: toPolicyInput(savePolicy) } }) - .then((result)=>{ + createPolicy({ variables: { input: toPolicyInput(savePolicy) } }).then((result) => { const newPolicy = { __typename: 'ListPoliciesResult', urn: result?.data?.createPolicy, @@ -213,13 +214,13 @@ export function usePolicy( setTimeout(() => { policiesRefetch(); }, 1000); - updateListPoliciesCache(client,newPolicy,DEFAULT_PAGE_SIZE); + updateListPoliciesCache(client, newPolicy, DEFAULT_PAGE_SIZE); onClosePolicyBuilder(); - }) + }); } }; - return{ + return { createPolicyError, updatePolicyError, deletePolicyError, @@ -227,5 +228,5 @@ export function usePolicy( onToggleActiveDuplicate, onRemovePolicy, getPrivilegeNames, - } -} \ No newline at end of file + }; +} diff --git a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx index 4c25bb8a8f588..ee29e92e06a22 100644 --- a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx +++ b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx @@ -269,7 +269,7 @@ export default function DefaultPreviewCard({ return ( - + {shouldShowRightColumn && ( - + {topUsers && topUsers?.length > 0 && ( <> diff --git a/datahub-web-react/src/app/search/SearchablePage.tsx b/datahub-web-react/src/app/search/SearchablePage.tsx index 53dfc866b9b64..8a2b0b35ec31f 100644 --- a/datahub-web-react/src/app/search/SearchablePage.tsx +++ b/datahub-web-react/src/app/search/SearchablePage.tsx @@ -75,22 +75,22 @@ export const SearchablePage = ({ onSearch, onAutoComplete, children }: Props) => useEffect(() => { // Update the title only if it's not already set and there is a valid pathname if (!title && location.pathname) { - const formattedPath = location.pathname - .split('/') - .filter(word => word !== '') - .map(word => word.charAt(0).toUpperCase() + word.slice(1)) - .join(' | '); - - if (formattedPath) { - return updateTitle(formattedPath); - } + const formattedPath = location.pathname + .split('/') + .filter((word) => word !== '') + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' | '); + + if (formattedPath) { + return updateTitle(formattedPath); + } } - + // Clean up the title when the component unmounts return () => { - updateTitle(''); + updateTitle(''); }; - }, [location.pathname, title, updateTitle]); + }, [location.pathname, title, updateTitle]); useEffect(() => { if (suggestionsData !== undefined) { diff --git a/datahub-web-react/src/app/search/filters/useSearchFilterDropdown.tsx b/datahub-web-react/src/app/search/filters/useSearchFilterDropdown.tsx index 752b615ae6462..8467449ab6029 100644 --- a/datahub-web-react/src/app/search/filters/useSearchFilterDropdown.tsx +++ b/datahub-web-react/src/app/search/filters/useSearchFilterDropdown.tsx @@ -39,8 +39,9 @@ export default function useSearchFilterDropdown({ filter, activeFilters, onChang useMemo(() => [filter.field], [filter.field]), ); const [aggregateAcrossEntities, { data, loading }] = useAggregateAcrossEntitiesLazyQuery(); - const [autoCompleteResults, setAutoCompleteResults] = - useState(undefined); + const [autoCompleteResults, setAutoCompleteResults] = useState( + undefined, + ); const [getAutoCompleteResults] = useGetAutoCompleteMultipleResultsLazyQuery({ onCompleted: (result) => setAutoCompleteResults(result), }); diff --git a/datahub-web-react/src/app/search/filters/utils.tsx b/datahub-web-react/src/app/search/filters/utils.tsx index 401b685732667..f115277a04967 100644 --- a/datahub-web-react/src/app/search/filters/utils.tsx +++ b/datahub-web-react/src/app/search/filters/utils.tsx @@ -369,4 +369,4 @@ export function useElementDimensions(ref) { }, [ref]); return dimensions; -} \ No newline at end of file +} diff --git a/datahub-web-react/src/app/search/utils/constants.ts b/datahub-web-react/src/app/search/utils/constants.ts index af45129022cc1..8616f67ac2b6a 100644 --- a/datahub-web-react/src/app/search/utils/constants.ts +++ b/datahub-web-react/src/app/search/utils/constants.ts @@ -120,6 +120,6 @@ export const FilterModes = { ADVANCED: 'advanced', } as const; -export type FilterMode = typeof FilterModes[keyof typeof FilterModes]; +export type FilterMode = (typeof FilterModes)[keyof typeof FilterModes]; export const MAX_COUNT_VAL = 10000; diff --git a/datahub-web-react/src/app/search/utils/csvUtils.ts b/datahub-web-react/src/app/search/utils/csvUtils.ts index f701b90ab78e4..cb29f5e486824 100644 --- a/datahub-web-react/src/app/search/utils/csvUtils.ts +++ b/datahub-web-react/src/app/search/utils/csvUtils.ts @@ -1,5 +1,5 @@ export function downloadFile(data: string, title: string) { - const blobx = new Blob([data], { type: 'text/plain' }); // ! Blob + const blobx = new Blob([data], { type: 'text/plain;chartset=utf-8' }); // ! Blob const elemx = window.document.createElement('a'); elemx.href = window.URL.createObjectURL(blobx); // ! createObjectURL elemx.download = title; diff --git a/datahub-web-react/src/app/shared/BrowserTabTitleContext.tsx b/datahub-web-react/src/app/shared/BrowserTabTitleContext.tsx index 284e2771124c8..61321d41f9986 100644 --- a/datahub-web-react/src/app/shared/BrowserTabTitleContext.tsx +++ b/datahub-web-react/src/app/shared/BrowserTabTitleContext.tsx @@ -1,30 +1,26 @@ import React, { createContext, ReactNode, useContext } from 'react'; interface BrowserTitleContextProps { - title: string; - updateTitle: (newTitle: string) => void; + title: string; + updateTitle: (newTitle: string) => void; } const BrowserTitleContext = createContext(undefined); export const BrowserTitleProvider: React.FC<{ children: ReactNode }> = ({ children }) => { - const [title, setTitle] = React.useState(''); + const [title, setTitle] = React.useState(''); - const updateTitle = (newTitle: string) => { - setTitle(newTitle); - }; + const updateTitle = (newTitle: string) => { + setTitle(newTitle); + }; - return ( - - {children} - - ); + return {children}; }; export const useBrowserTitle = () => { - const context = useContext(BrowserTitleContext); - if (!context) { - throw new Error('useBrowserTitle must be used within a BrowserTitleProvider'); - } - return context; + const context = useContext(BrowserTitleContext); + if (!context) { + throw new Error('useBrowserTitle must be used within a BrowserTitleProvider'); + } + return context; }; diff --git a/datahub-web-react/src/app/shared/ShowMoreSection.tsx b/datahub-web-react/src/app/shared/ShowMoreSection.tsx new file mode 100644 index 0000000000000..d315dd5b8d1ce --- /dev/null +++ b/datahub-web-react/src/app/shared/ShowMoreSection.tsx @@ -0,0 +1,31 @@ +import React from 'react'; +import styled from 'styled-components'; +import { ANTD_GRAY } from '../entity/shared/constants'; + +export const ShowMoreButton = styled.div` + margin-top: 12px; + padding: 0px; + color: ${ANTD_GRAY[7]}; + text-align: left; + :hover { + cursor: pointer; + color: ${ANTD_GRAY[8]}; + text-decoration: underline; + } +`; + +type Props = { + totalCount: number; + visibleCount: number; + setVisibleCount: (visibleCount: number) => void; + pageSize?: number; +}; + +export const ShowMoreSection = ({ totalCount, visibleCount, setVisibleCount, pageSize = 4 }: Props) => { + const showMoreCount = visibleCount + pageSize > totalCount ? totalCount - visibleCount : pageSize; + return ( + setVisibleCount(visibleCount + pageSize)}> + {showMoreCount ? `show ${showMoreCount} more` : 'show more'} + + ); +}; diff --git a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx index 467e535f9bad4..e7b0025118ff1 100644 --- a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx +++ b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx @@ -122,20 +122,22 @@ export function HeaderLinks(props: Props) { Manage related groups of data assets - {businessAttributesFlag && ( - - - - Business Attribute - - Universal field for data consistency - - )} + {businessAttributesFlag && ( + + + + + Business Attribute + + Universal field for data consistency + + + )} } > diff --git a/datahub-web-react/src/app/shared/avatar/CustomAvatar.tsx b/datahub-web-react/src/app/shared/avatar/CustomAvatar.tsx index 2ec5a1e77fe22..994012d2edb9e 100644 --- a/datahub-web-react/src/app/shared/avatar/CustomAvatar.tsx +++ b/datahub-web-react/src/app/shared/avatar/CustomAvatar.tsx @@ -66,13 +66,17 @@ export default function CustomAvatar({ ); const handleImageError = () => { - setImageError(true); - // To prevent fallback error handling from Ant Design - return false; + setImageError(true); + // To prevent fallback error handling from Ant Design + return false; }; const avatar = - photoUrl && photoUrl !== '' && !imageError ? : avatarWithDefault; + photoUrl && photoUrl !== '' && !imageError ? ( + + ) : ( + avatarWithDefault + ); if (!name) { return url ? {avatar} : avatar; } diff --git a/datahub-web-react/src/app/shared/businessAttribute/AddBusinessAttributeModal.tsx b/datahub-web-react/src/app/shared/businessAttribute/AddBusinessAttributeModal.tsx index 88f6a4c9660d3..5ba0536af1bb3 100644 --- a/datahub-web-react/src/app/shared/businessAttribute/AddBusinessAttributeModal.tsx +++ b/datahub-web-react/src/app/shared/businessAttribute/AddBusinessAttributeModal.tsx @@ -229,11 +229,13 @@ export default function EditBusinessAttributeModal({ variables: { input: { businessAttributeUrn: urn, - resourceUrn: [{ - resourceUrn: resources[0].resourceUrn, - subResource: resources[0].subResource, - subResourceType: resources[0].subResourceType, - }], + resourceUrn: [ + { + resourceUrn: resources[0].resourceUrn, + subResource: resources[0].subResource, + subResourceType: resources[0].subResourceType, + }, + ], }, }, }) diff --git a/datahub-web-react/src/app/shared/businessAttribute/AttributeContent.tsx b/datahub-web-react/src/app/shared/businessAttribute/AttributeContent.tsx index 61306c9cf64d3..41ecbe69b57fc 100644 --- a/datahub-web-react/src/app/shared/businessAttribute/AttributeContent.tsx +++ b/datahub-web-react/src/app/shared/businessAttribute/AttributeContent.tsx @@ -67,11 +67,13 @@ export default function AttributeContent({ variables: { input: { businessAttributeUrn: attributeToRemove.businessAttribute.urn, - resourceUrn: [{ - resourceUrn: attributeToRemove.associatedUrn || entityUrn || '', - subResource: null, - subResourceType: null, - }], + resourceUrn: [ + { + resourceUrn: attributeToRemove.associatedUrn || entityUrn || '', + subResource: null, + subResourceType: null, + }, + ], }, }, }) diff --git a/datahub-web-react/src/app/shared/form/RequiredFieldForm.tsx b/datahub-web-react/src/app/shared/form/RequiredFieldForm.tsx new file mode 100644 index 0000000000000..d35af17e1b1ce --- /dev/null +++ b/datahub-web-react/src/app/shared/form/RequiredFieldForm.tsx @@ -0,0 +1,14 @@ +import { Form } from 'antd'; +import styled from 'styled-components'; + +const DEFAULT_ASTERICK_COLOR = '#F5222D'; + +export const RequiredFieldForm = styled(Form)<{ requiredColor?: string }>` + && { + .ant-form-item-label > label.ant-form-item-required::before { + color: ${(props) => + props.requiredColor || DEFAULT_ASTERICK_COLOR}; /* Change 'red' to any color you prefer */ + content: '*'; /* Ensure the asterisk is always used */ + } + } +`; diff --git a/datahub-web-react/src/app/shared/numberUtil.ts b/datahub-web-react/src/app/shared/numberUtil.ts new file mode 100644 index 0000000000000..82280c75b47f8 --- /dev/null +++ b/datahub-web-react/src/app/shared/numberUtil.ts @@ -0,0 +1,22 @@ +export function parseMaybeStringAsFloatOrDefault(str: any, fallback?: T): number | T | undefined { + const parsedValue = typeof str === 'string' ? parseFloat(str) : str; + return typeof parsedValue === 'number' && !Number.isNaN(parsedValue) ? parsedValue : fallback; +} + +export function parseJsonArrayOrDefault(str: any, fallback: T[] = []): T[] | undefined { + // Check if the input is a string and try to parse it. + if (typeof str === 'string') { + try { + const parsedValue = JSON.parse(str); + // Check if the parsed value is an array before returning. + if (Array.isArray(parsedValue)) { + return parsedValue; + } + } catch (e) { + // If parsing throws, log the error (optional) and proceed to return fallback. + console.error('Failed to parse JSON:', e); + } + } + // Return fallback if the above conditions fail. + return fallback; +} diff --git a/datahub-web-react/src/appConfigContext.tsx b/datahub-web-react/src/appConfigContext.tsx index b4f16e2d2a824..4262f772b006e 100644 --- a/datahub-web-react/src/appConfigContext.tsx +++ b/datahub-web-react/src/appConfigContext.tsx @@ -53,6 +53,7 @@ export const DEFAULT_APP_CONFIG = { nestedDomainsEnabled: true, platformBrowseV2: false, businessAttributeEntityEnabled: false, + dataContractsEnabled: false, }, }; diff --git a/datahub-web-react/src/conf/analytics.ts b/datahub-web-react/src/conf/analytics.ts index 2506ec408b5ba..b516c1babba74 100644 --- a/datahub-web-react/src/conf/analytics.ts +++ b/datahub-web-react/src/conf/analytics.ts @@ -1,7 +1,7 @@ const config: any = { // Uncomment below to configure analytics. // googleAnalytics: { - // trackingId: 'UA-24123123-01', + // measurementIds: ['G-RTEGER123'], // }, // mixpanel: { // token: 'fad1285da4e618b618973cacf6565e61', diff --git a/datahub-web-react/src/conf/theme/theme_dark.config.json b/datahub-web-react/src/conf/theme/theme_dark.config.json index 54ebebd3b692b..d2c8912ac1f50 100644 --- a/datahub-web-react/src/conf/theme/theme_dark.config.json +++ b/datahub-web-react/src/conf/theme/theme_dark.config.json @@ -52,4 +52,4 @@ ] } } -} \ No newline at end of file +} diff --git a/datahub-web-react/src/conf/theme/theme_light.config.json b/datahub-web-react/src/conf/theme/theme_light.config.json index 6b9ef3eac52b0..9983c5a8b67b7 100644 --- a/datahub-web-react/src/conf/theme/theme_light.config.json +++ b/datahub-web-react/src/conf/theme/theme_light.config.json @@ -60,4 +60,4 @@ ] } } -} \ No newline at end of file +} diff --git a/datahub-web-react/src/graphql/accessrole.graphql b/datahub-web-react/src/graphql/accessrole.graphql index ccc7d3496ad6d..ad539f24e2bc3 100644 --- a/datahub-web-react/src/graphql/accessrole.graphql +++ b/datahub-web-react/src/graphql/accessrole.graphql @@ -5,4 +5,4 @@ query getExternalRole($urn: String!) { description } } -} \ No newline at end of file +} diff --git a/datahub-web-react/src/graphql/app.graphql b/datahub-web-react/src/graphql/app.graphql index 7b47fc0302247..662b2f11336fa 100644 --- a/datahub-web-react/src/graphql/app.graphql +++ b/datahub-web-react/src/graphql/app.graphql @@ -68,6 +68,7 @@ query appConfig { nestedDomainsEnabled platformBrowseV2 businessAttributeEntityEnabled + dataContractsEnabled } } } diff --git a/datahub-web-react/src/graphql/assertion.graphql b/datahub-web-react/src/graphql/assertion.graphql index 016e8a4f06086..af4f0f73f90ce 100644 --- a/datahub-web-react/src/graphql/assertion.graphql +++ b/datahub-web-react/src/graphql/assertion.graphql @@ -1,3 +1,173 @@ +fragment assertionInfo on AssertionInfo { + type + description + lastUpdated { + time + actor + } + datasetAssertion { + datasetUrn + scope + aggregation + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + fields { + urn + path + } + nativeType + nativeParameters { + key + value + } + logic + } + freshnessAssertion { + type + entityUrn + schedule { + type + cron { + cron + timezone + } + fixedInterval { + unit + multiple + } + } + filter { + type + sql + } + } + sqlAssertion { + type + entityUrn + statement + changeType + operator + parameters { + ...assertionStdParametersDetails + } + } + fieldAssertion { + type + entityUrn + filter { + type + sql + } + fieldValuesAssertion { + field { + path + type + nativeType + } + transform { + type + } + operator + parameters { + ...assertionStdParametersDetails + } + failThreshold { + type + value + } + excludeNulls + } + fieldMetricAssertion { + field { + path + type + nativeType + } + metric + operator + parameters { + ...assertionStdParametersDetails + } + } + } + volumeAssertion { + type + entityUrn + filter { + type + sql + } + rowCountTotal { + operator + parameters { + ...assertionStdParametersDetails + } + } + rowCountChange { + type + operator + parameters { + ...assertionStdParametersDetails + } + } + incrementingSegmentRowCountTotal { + operator + parameters { + ...assertionStdParametersDetails + } + segment { + ...incrementingSegmentSpecDetails + } + } + incrementingSegmentRowCountChange { + type + operator + parameters { + ...assertionStdParametersDetails + } + segment { + ...incrementingSegmentSpecDetails + } + } + } + schemaAssertion { + entityUrn + compatibility + fields { + path + type + nativeType + } + schema { + fields { + fieldPath + type + nativeDataType + } + } + } + source { + type + created { + time + actor + } + } +} + fragment assertionDetails on Assertion { urn type @@ -16,47 +186,46 @@ fragment assertionDetails on Assertion { ...dataPlatformInstanceFields } info { - type - datasetAssertion { - scope - aggregation - operator - parameters { - value { - value - type - } - minValue { - value - type - } - maxValue { - value - type - } - } - fields { - urn - path - } - nativeType - nativeParameters { - key - value - } - logic - } - description + ...assertionInfo } tags { ...globalTagsFields } } +fragment incrementingSegmentSpecDetails on IncrementingSegmentSpec { + field { + path + type + nativeType + } + transformer { + type + nativeType + } +} + +fragment assertionStdParametersDetails on AssertionStdParameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } +} + fragment assertionRunEventDetails on AssertionRunEvent { timestampMillis + lastObservedMillis assertionUrn status + runId runtimeContext { key value @@ -93,3 +262,15 @@ query getAssertionRuns($assertionUrn: String!, $startTime: Long, $endTime: Long, mutation deleteAssertion($urn: String!) { deleteAssertion(urn: $urn) } + +fragment assertionDetailsWithRunEvents on Assertion { + ...assertionDetails + runEvents(status: COMPLETE, limit: 1) { + total + failed + succeeded + runEvents { + ...assertionRunEventDetails + } + } +} diff --git a/datahub-web-react/src/graphql/auth.graphql b/datahub-web-react/src/graphql/auth.graphql index 3bcb411471d62..b0a0df0abb2e0 100644 --- a/datahub-web-react/src/graphql/auth.graphql +++ b/datahub-web-react/src/graphql/auth.graphql @@ -24,11 +24,11 @@ query listAccessTokens($input: ListAccessTokenInput!) { } mutation createAccessToken($input: CreateAccessTokenInput!) { - createAccessToken(input: $input){ + createAccessToken(input: $input) { accessToken } } mutation revokeAccessToken($tokenId: String!) { revokeAccessToken(tokenId: $tokenId) -} \ No newline at end of file +} diff --git a/datahub-web-react/src/graphql/browseV2.graphql b/datahub-web-react/src/graphql/browseV2.graphql index 2846dd3940416..5316fd2b84e9c 100644 --- a/datahub-web-react/src/graphql/browseV2.graphql +++ b/datahub-web-react/src/graphql/browseV2.graphql @@ -11,7 +11,6 @@ query getBrowseResultsV2($input: BrowseV2Input!) { properties { name } - } ... on DataFlow { properties { diff --git a/datahub-web-react/src/graphql/container.graphql b/datahub-web-react/src/graphql/container.graphql index efeca6da5e6dd..4b3ecfe8aaaff 100644 --- a/datahub-web-react/src/graphql/container.graphql +++ b/datahub-web-react/src/graphql/container.graphql @@ -1,6 +1,7 @@ query getContainer($urn: String!) { container(urn: $urn) { urn + type exists lastIngested platform { @@ -9,6 +10,7 @@ query getContainer($urn: String!) { properties { name description + externalUrl customProperties { key value diff --git a/datahub-web-react/src/graphql/contract.graphql b/datahub-web-react/src/graphql/contract.graphql new file mode 100644 index 0000000000000..4c54dba037a35 --- /dev/null +++ b/datahub-web-react/src/graphql/contract.graphql @@ -0,0 +1,53 @@ +fragment contractAssertionParameters on AssertionStdParameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } +} + +fragment dataContractDetails on DataContract { + urn + type + properties { + freshness { + assertion { + ...assertionDetailsWithRunEvents + } + } + schema { + assertion { + ...assertionDetailsWithRunEvents + } + } + dataQuality { + assertion { + ...assertionDetailsWithRunEvents + } + } + } + status { + state + } +} + +query getDatasetContract($urn: String!) { + dataset(urn: $urn) { + contract { + ...dataContractDetails + } + } +} + +mutation upsertDataContract($input: UpsertDataContractInput!) { + upsertDataContract(input: $input) { + ...dataContractDetails + } +} diff --git a/datahub-web-react/src/graphql/dashboard.graphql b/datahub-web-react/src/graphql/dashboard.graphql index 68a966a68e00a..681c98f361ccb 100644 --- a/datahub-web-react/src/graphql/dashboard.graphql +++ b/datahub-web-react/src/graphql/dashboard.graphql @@ -1,5 +1,7 @@ query getDashboard($urn: String!) { dashboard(urn: $urn) { + urn + type ...dashboardFields privileges { ...entityPrivileges diff --git a/datahub-web-react/src/graphql/dataJob.graphql b/datahub-web-react/src/graphql/dataJob.graphql index 78247bd460fbb..836aac35deaf5 100644 --- a/datahub-web-react/src/graphql/dataJob.graphql +++ b/datahub-web-react/src/graphql/dataJob.graphql @@ -1,5 +1,7 @@ query getDataJob($urn: String!) { dataJob(urn: $urn) { + urn + type ...dataJobFields privileges { ...entityPrivileges diff --git a/datahub-web-react/src/graphql/dataPlatform.graphql b/datahub-web-react/src/graphql/dataPlatform.graphql index 6281cf155a5d2..44acbf6737fae 100644 --- a/datahub-web-react/src/graphql/dataPlatform.graphql +++ b/datahub-web-react/src/graphql/dataPlatform.graphql @@ -1,5 +1,7 @@ query getDataPlatform($urn: String!) { dataPlatform(urn: $urn) { + urn + type ...platformFields } } diff --git a/datahub-web-react/src/graphql/dataProduct.graphql b/datahub-web-react/src/graphql/dataProduct.graphql index eb053ca956131..623ece13dbfc1 100644 --- a/datahub-web-react/src/graphql/dataProduct.graphql +++ b/datahub-web-react/src/graphql/dataProduct.graphql @@ -1,5 +1,7 @@ query getDataProduct($urn: String!) { dataProduct(urn: $urn) { + urn + type ...dataProductFields privileges { ...entityPrivileges diff --git a/datahub-web-react/src/graphql/dataset.graphql b/datahub-web-react/src/graphql/dataset.graphql index 1ca25a6ba3bf6..fcca919f61423 100644 --- a/datahub-web-react/src/graphql/dataset.graphql +++ b/datahub-web-react/src/graphql/dataset.graphql @@ -1,6 +1,7 @@ query getDataProfiles($urn: String!, $limit: Int, $startTime: Long, $endTime: Long) { dataset(urn: $urn) { urn + type datasetProfiles(limit: $limit, startTimeMillis: $startTime, endTimeMillis: $endTime) { rowCount columnCount diff --git a/datahub-web-react/src/graphql/ermodelrelationship.graphql b/datahub-web-react/src/graphql/ermodelrelationship.graphql index c3163c7c21ffe..f7f42cb51ae01 100644 --- a/datahub-web-react/src/graphql/ermodelrelationship.graphql +++ b/datahub-web-react/src/graphql/ermodelrelationship.graphql @@ -37,7 +37,6 @@ query getERModelRelationship($urn: String!) { ... on ERModelRelationship { urn } - } } } @@ -54,4 +53,4 @@ mutation createERModelRelationship($input: ERModelRelationshipUpdateInput!) { type id } -} \ No newline at end of file +} diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index b28150a47b753..e8b621df649a9 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -990,9 +990,7 @@ fragment domainEntitiesFields on Domain { entities(input: { start: 0, count: 0 }) { total } - dataProducts: entities( - input: { start: 0, count: 0, filters: [{ field: "_entityType", value: "DATA_PRODUCT" }] } - ) { + dataProducts: entities(input: { start: 0, count: 0, filters: [{ field: "_entityType", value: "DATA_PRODUCT" }] }) { total } children: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 0 }) { @@ -1382,7 +1380,6 @@ fragment entityHealth on Health { causes } - fragment formsFields on Forms { completedForms { ...formAssociationFields @@ -1418,9 +1415,9 @@ fragment businessAttribute on BusinessAttributeAssociation { businessAttribute { urn type - ownership { - ...ownershipFields - } + ownership { + ...ownershipFields + } properties { name description diff --git a/datahub-web-react/src/graphql/glossaryNode.graphql b/datahub-web-react/src/graphql/glossaryNode.graphql index 6e232827c2370..bc260c8d5365f 100644 --- a/datahub-web-react/src/graphql/glossaryNode.graphql +++ b/datahub-web-react/src/graphql/glossaryNode.graphql @@ -37,14 +37,7 @@ query getGlossaryNode($urn: String!) { forms { ...formsFields } - children: relationships( - input: { - types: ["IsPartOf"] - direction: INCOMING - start: 0 - count: 10000 - } - ) { + children: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 10000 }) { total relationships { direction diff --git a/datahub-web-react/src/graphql/incident.graphql b/datahub-web-react/src/graphql/incident.graphql index 3ee18df9d6e94..c6708ab894e93 100644 --- a/datahub-web-react/src/graphql/incident.graphql +++ b/datahub-web-react/src/graphql/incident.graphql @@ -30,7 +30,6 @@ fragment incidentsFields on EntityIncidentsResult { } } - fragment datasetSiblingIncidents on Dataset { siblings { isPrimary @@ -75,4 +74,4 @@ query getEntityIncidents($urn: String!, $start: Int!, $count: Int!, $state: Inci } } } -} \ No newline at end of file +} diff --git a/datahub-web-react/src/graphql/ingestion.graphql b/datahub-web-react/src/graphql/ingestion.graphql index c172ccdbe7632..ebdfb0cfd87e6 100644 --- a/datahub-web-react/src/graphql/ingestion.graphql +++ b/datahub-web-react/src/graphql/ingestion.graphql @@ -39,6 +39,10 @@ query listIngestionSources($input: ListIngestionSourcesInput!) { status startTimeMs durationMs + structuredReport { + type + serializedValue + } } } } @@ -86,6 +90,10 @@ query getIngestionSource($urn: String!, $runStart: Int, $runCount: Int) { status startTimeMs durationMs + structuredReport { + type + serializedValue + } } } } diff --git a/datahub-web-react/src/graphql/lineage.graphql b/datahub-web-react/src/graphql/lineage.graphql index 612adcc5f777d..c4936fc3c8abe 100644 --- a/datahub-web-react/src/graphql/lineage.graphql +++ b/datahub-web-react/src/graphql/lineage.graphql @@ -176,7 +176,7 @@ fragment lineageNodeProperties on EntityWithRelationships { ...entityDomain } parentContainers { - ...parentContainersFields + ...parentContainersFields } ...entityDataProduct status { @@ -206,7 +206,7 @@ fragment lineageNodeProperties on EntityWithRelationships { ...ownershipFields } parentContainers { - ...parentContainersFields + ...parentContainersFields } subTypes { typeNames @@ -466,12 +466,12 @@ query getEntityLineage( $excludeDownstream: Boolean = false ) { entity(urn: $urn) { - ...entityLineage + ...entityLineage } } query getBulkEntityLineage( - $urns: [String!]!, + $urns: [String!]! $separateSiblings: Boolean $showColumns: Boolean! $startTimeMillis: Long diff --git a/datahub-web-react/src/graphql/me.graphql b/datahub-web-react/src/graphql/me.graphql index 9a1fb89a249eb..549aae1ad17de 100644 --- a/datahub-web-react/src/graphql/me.graphql +++ b/datahub-web-react/src/graphql/me.graphql @@ -51,7 +51,6 @@ query getMe { manageGlobalAnnouncements createBusinessAttributes manageBusinessAttributes - } } } diff --git a/datahub-web-react/src/graphql/mlFeature.graphql b/datahub-web-react/src/graphql/mlFeature.graphql index d6a75e16b86f1..2ed5ecfb37fda 100644 --- a/datahub-web-react/src/graphql/mlFeature.graphql +++ b/datahub-web-react/src/graphql/mlFeature.graphql @@ -1,5 +1,7 @@ query getMLFeature($urn: String!) { mlFeature(urn: $urn) { + urn + type ...nonRecursiveMLFeature privileges { ...entityPrivileges diff --git a/datahub-web-react/src/graphql/mlFeatureTable.graphql b/datahub-web-react/src/graphql/mlFeatureTable.graphql index a6e069c120518..02efbaf9766e1 100644 --- a/datahub-web-react/src/graphql/mlFeatureTable.graphql +++ b/datahub-web-react/src/graphql/mlFeatureTable.graphql @@ -1,5 +1,7 @@ query getMLFeatureTable($urn: String!) { mlFeatureTable(urn: $urn) { + urn + type ...nonRecursiveMLFeatureTable privileges { ...entityPrivileges diff --git a/datahub-web-react/src/graphql/mlModel.graphql b/datahub-web-react/src/graphql/mlModel.graphql index 1626bc473213a..2192888caef70 100644 --- a/datahub-web-react/src/graphql/mlModel.graphql +++ b/datahub-web-react/src/graphql/mlModel.graphql @@ -1,5 +1,7 @@ query getMLModel($urn: String!) { mlModel(urn: $urn) { + urn + type ...nonRecursiveMLModel features: relationships(input: { types: ["Consumes"], direction: OUTGOING, start: 0, count: 100 }) { start diff --git a/datahub-web-react/src/graphql/mlModelGroup.graphql b/datahub-web-react/src/graphql/mlModelGroup.graphql index 8ae049c8c0b1d..81ab65d0b9a08 100644 --- a/datahub-web-react/src/graphql/mlModelGroup.graphql +++ b/datahub-web-react/src/graphql/mlModelGroup.graphql @@ -1,5 +1,7 @@ query getMLModelGroup($urn: String!) { mlModelGroup(urn: $urn) { + urn + type ...nonRecursiveMLModelGroupFields incoming: relationships( input: { diff --git a/datahub-web-react/src/graphql/mlPrimaryKey.graphql b/datahub-web-react/src/graphql/mlPrimaryKey.graphql index 599c4d7fabcac..d39f9d3fbdfa2 100644 --- a/datahub-web-react/src/graphql/mlPrimaryKey.graphql +++ b/datahub-web-react/src/graphql/mlPrimaryKey.graphql @@ -1,5 +1,7 @@ query getMLPrimaryKey($urn: String!) { mlPrimaryKey(urn: $urn) { + urn + type ...nonRecursiveMLPrimaryKey privileges { ...entityPrivileges diff --git a/datahub-web-react/src/graphql/ownership.graphql b/datahub-web-react/src/graphql/ownership.graphql index 634873f1e529b..b0008f9005f07 100644 --- a/datahub-web-react/src/graphql/ownership.graphql +++ b/datahub-web-react/src/graphql/ownership.graphql @@ -43,4 +43,4 @@ fragment ownershipTypeFields on OwnershipTypeEntity { status { removed } -} \ No newline at end of file +} diff --git a/datahub-web-react/src/graphql/preview.graphql b/datahub-web-react/src/graphql/preview.graphql index 48482c64face9..e675f142a1eeb 100644 --- a/datahub-web-react/src/graphql/preview.graphql +++ b/datahub-web-react/src/graphql/preview.graphql @@ -156,7 +156,7 @@ fragment entityPreview on Entity { } health { ...entityHealth - } + } } ... on DataFlow { urn diff --git a/datahub-web-react/src/graphql/relationships.graphql b/datahub-web-react/src/graphql/relationships.graphql index 7d6dd28dfe029..e2afff5dfa6d2 100644 --- a/datahub-web-react/src/graphql/relationships.graphql +++ b/datahub-web-react/src/graphql/relationships.graphql @@ -42,4 +42,4 @@ fragment relationshipFields on EntityWithRelationships { downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) { ...leafLineageResults } -} \ No newline at end of file +} diff --git a/datahub-web-react/src/graphql/schemaBlame.graphql b/datahub-web-react/src/graphql/schemaBlame.graphql index 4fafaa4340c88..a2dbba5b1ae18 100644 --- a/datahub-web-react/src/graphql/schemaBlame.graphql +++ b/datahub-web-react/src/graphql/schemaBlame.graphql @@ -30,4 +30,4 @@ query getSchemaVersionList($input: GetSchemaVersionListInput!) { versionStamp } } -} \ No newline at end of file +} diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index b8dc84aea62a7..b0acc454fa55b 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -859,7 +859,7 @@ fragment searchResultsWithoutSchemaField on Entity { fragment searchResultFields on Entity { ...searchResultsWithoutSchemaField ... on SchemaFieldEntity { - ...entityField + ...entityField } } @@ -969,23 +969,23 @@ fragment searchResults on SearchResults { } fragment entityField on SchemaFieldEntity { - urn - type - parent { urn type - } - fieldPath - structuredProperties { - properties { - ...structuredPropertiesFields + parent { + urn + type } - } - businessAttributes { - businessAttribute { - ...businessAttribute + fieldPath + structuredProperties { + properties { + ...structuredPropertiesFields + } + } + businessAttributes { + businessAttribute { + ...businessAttribute + } } - } } fragment schemaFieldEntityFields on SchemaFieldEntity { diff --git a/datahub-web-react/src/graphql/tag.graphql b/datahub-web-react/src/graphql/tag.graphql index 031d923276bfe..0bf0953b15fbe 100644 --- a/datahub-web-react/src/graphql/tag.graphql +++ b/datahub-web-react/src/graphql/tag.graphql @@ -1,6 +1,7 @@ query getTag($urn: String!) { tag(urn: $urn) { urn + type name description properties { diff --git a/datahub-web-react/src/graphql/user.graphql b/datahub-web-react/src/graphql/user.graphql index a8a4e90284956..030ef85df7124 100644 --- a/datahub-web-react/src/graphql/user.graphql +++ b/datahub-web-react/src/graphql/user.graphql @@ -1,6 +1,7 @@ query getUser($urn: String!, $groupsCount: Int!) { corpUser(urn: $urn) { urn + type username isNativeUser exists diff --git a/datahub-web-react/src/images/db2logo.png b/datahub-web-react/src/images/db2logo.png new file mode 100644 index 0000000000000..bc842aa418b4e Binary files /dev/null and b/datahub-web-react/src/images/db2logo.png differ diff --git a/datahub-web-react/src/images/verticalogo copy.png b/datahub-web-react/src/images/verticalogo copy.png deleted file mode 100644 index 5da38f4e67c7d..0000000000000 Binary files a/datahub-web-react/src/images/verticalogo copy.png and /dev/null differ diff --git a/datahub-web-react/src/utils/focus/index.ts b/datahub-web-react/src/utils/focus/index.ts index 313c00f429018..9638e080d77b3 100644 --- a/datahub-web-react/src/utils/focus/index.ts +++ b/datahub-web-react/src/utils/focus/index.ts @@ -3,4 +3,4 @@ // The getContainer prop allows you to specify the container in which the modal should be rendered. // By default, modals are appended to the end of the document body, but using getContainer, you can specify a different container. export const getModalDomContainer = () => document.getElementById('root') as HTMLElement; -// this can we remove once we upgrade to new antd version i.e 5.11.2 because there we have autoFocus property for the modal. \ No newline at end of file +// this can we remove once we upgrade to new antd version i.e 5.11.2 because there we have autoFocus property for the modal. diff --git a/datahub-web-react/src/utils/test-utils/TestPageContainer.tsx b/datahub-web-react/src/utils/test-utils/TestPageContainer.tsx index c07505d7315a1..eaca969bb524e 100644 --- a/datahub-web-react/src/utils/test-utils/TestPageContainer.tsx +++ b/datahub-web-react/src/utils/test-utils/TestPageContainer.tsx @@ -25,7 +25,7 @@ import UserContextProvider from '../../app/context/UserContextProvider'; import { DataPlatformEntity } from '../../app/entity/dataPlatform/DataPlatformEntity'; import { ContainerEntity } from '../../app/entity/container/ContainerEntity'; import AppConfigProvider from '../../AppConfigProvider'; -import {BusinessAttributeEntity} from "../../app/entity/businessAttribute/BusinessAttributeEntity"; +import { BusinessAttributeEntity } from '../../app/entity/businessAttribute/BusinessAttributeEntity'; type Props = { children: React.ReactNode; @@ -58,7 +58,7 @@ export default ({ children, initialEntries }: Props) => { writable: true, value: `${CLIENT_AUTH_COOKIE}=urn:li:corpuser:2`, }); - vi.mock('js-cookie', () => ({ default: { get: () => 'urn:li:corpuser:2' }})); + vi.mock('js-cookie', () => ({ default: { get: () => 'urn:li:corpuser:2' } })); return ( diff --git a/datahub-web-react/yarn.lock b/datahub-web-react/yarn.lock index aad6d84c6bd93..4268acf155de1 100644 --- a/datahub-web-react/yarn.lock +++ b/datahub-web-react/yarn.lock @@ -43,12 +43,10 @@ dependencies: "@analytics/type-utils" "^0.6.2" -"@analytics/google-analytics@^0.5.2": - version "0.5.2" - resolved "https://registry.yarnpkg.com/@analytics/google-analytics/-/google-analytics-0.5.2.tgz#e94501ae23284ebdccb3453cf9158984b7443388" - integrity sha512-6jviLzo6s7MLvsfPLCSppGgoHb9piShCk/n66s75yR8XN+Pz8ADk1Rh28jRErbLSJToJcCMOwDXikCF7U4exMQ== - dependencies: - universal-analytics "^0.4.20" +"@analytics/google-analytics@^1.0.7": + version "1.0.7" + resolved "https://registry.yarnpkg.com/@analytics/google-analytics/-/google-analytics-1.0.7.tgz#256658c841af707aabdcfb35858971676115d1ce" + integrity sha512-KZ69NaMIi5kOcouzqI8cu7tZgQl7ziGiRahfU6zniUf32G8bv7wQDh73JFz1NwO6gBPloUc+5BzEoWzScM5Rgw== "@analytics/localstorage-utils@^0.1.10": version "0.1.10" @@ -4114,7 +4112,7 @@ aggregate-error@^3.0.0: clean-stack "^2.0.0" indent-string "^4.0.0" -ajv@^6.10.0, ajv@^6.12.3, ajv@^6.12.4: +ajv@^6.10.0, ajv@^6.12.4: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -4319,13 +4317,6 @@ asap@~2.0.3: resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY= -asn1@~0.2.3: - version "0.2.4" - resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" - integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg== - dependencies: - safer-buffer "~2.1.0" - asn1js@^3.0.1, asn1js@^3.0.5: version "3.0.5" resolved "https://registry.yarnpkg.com/asn1js/-/asn1js-3.0.5.tgz#5ea36820443dbefb51cc7f88a2ebb5b462114f38" @@ -4335,11 +4326,6 @@ asn1js@^3.0.1, asn1js@^3.0.5: pvutils "^1.1.3" tslib "^2.4.0" -assert-plus@1.0.0, assert-plus@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" - integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU= - assertion-error@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-1.1.0.tgz#e60b6b0e8f301bd97e5375215bda406c85118c0b" @@ -4380,16 +4366,6 @@ available-typed-arrays@^1.0.5: resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== -aws-sign2@~0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" - integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg= - -aws4@^1.8.0: - version "1.11.0" - resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" - integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== - axe-core@^4.6.2: version "4.7.2" resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.7.2.tgz#040a7342b20765cb18bb50b628394c21bccc17a0" @@ -4502,13 +4478,6 @@ base64-js@^1.3.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== -bcrypt-pbkdf@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" - integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4= - dependencies: - tweetnacl "^0.14.3" - binary-extensions@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" @@ -4669,11 +4638,6 @@ case-anything@^2.1.10: resolved "https://registry.yarnpkg.com/case-anything/-/case-anything-2.1.10.tgz#d18a6ca968d54ec3421df71e3e190f3bced23410" integrity sha512-JczJwVrCP0jPKh05McyVsuOg6AYosrB9XWZKbQzXeDAm2ClE/PJE/BcrrQrVyGYH7Jg8V/LDupmyL4kFlVsVFQ== -caseless@~0.12.0: - version "0.12.0" - resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" - integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= - ccount@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/ccount/-/ccount-1.1.0.tgz#246687debb6014735131be8abab2d93898f8d043" @@ -4935,7 +4899,7 @@ colorette@^2.0.16: resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.20.tgz#9eb793e6833067f7235902fcd3b09917a000a95a" integrity sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w== -combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: +combined-stream@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== @@ -5005,11 +4969,6 @@ copy-to-clipboard@^3.2.0, copy-to-clipboard@^3.3.1: dependencies: toggle-selection "^1.0.6" -core-util-is@1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" - integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= - cosmiconfig@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-6.0.0.tgz#da4fee853c52f6b1e6935f41c1a2fc50bd4a9982" @@ -5236,13 +5195,6 @@ dash-get@^1.0.2: resolved "https://registry.yarnpkg.com/dash-get/-/dash-get-1.0.2.tgz#4c9e9ad5ef04c4bf9d3c9a451f6f7997298dcc7c" integrity sha512-4FbVrHDwfOASx7uQVxeiCTo7ggSdYZbqs8lH+WU6ViypPlDbe9y6IP5VVUDQBv9DcnyaiPT5XT0UWHgJ64zLeQ== -dashdash@^1.12.0: - version "1.14.1" - resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" - integrity sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA= - dependencies: - assert-plus "^1.0.0" - data-urls@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-4.0.0.tgz#333a454eca6f9a5b7b0f1013ff89074c3f522dd4" @@ -5509,18 +5461,10 @@ duplexer@^0.1.2: resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.2.tgz#3abe43aef3835f8ae077d136ddce0f276b0400e6" integrity sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg== -ecc-jsbn@~0.1.1: - version "0.1.2" - resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" - integrity sha1-OoOpBOVDUyh4dMVkt1SThoSamMk= - dependencies: - jsbn "~0.1.0" - safer-buffer "^2.1.0" - ejs@^3.1.5: - version "3.1.9" - resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.9.tgz#03c9e8777fe12686a9effcef22303ca3d8eeb361" - integrity sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ== + version "3.1.10" + resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.10.tgz#69ab8358b14e896f80cc39e62087b88500c3ac3b" + integrity sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA== dependencies: jake "^10.8.5" @@ -5941,7 +5885,7 @@ esutils@^2.0.2: resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== -extend@^3.0.0, extend@~3.0.2: +extend@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== @@ -5965,16 +5909,6 @@ extract-files@^11.0.0: resolved "https://registry.yarnpkg.com/extract-files/-/extract-files-11.0.0.tgz#b72d428712f787eef1f5193aff8ab5351ca8469a" integrity sha512-FuoE1qtbJ4bBVvv94CC7s0oTnKUGvQs+Rjf1L2SJFfS+HTVVjhPFtehPdQ0JiGPqVNfSSZvL5yzHHQq2Z4WNhQ== -extsprintf@1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" - integrity sha1-lpGEQOMEGnpBT4xS48V06zw+HgU= - -extsprintf@^1.2.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" - integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8= - faker@5.5.3: version "5.5.3" resolved "https://registry.npmjs.org/faker/-/faker-5.5.3.tgz#c57974ee484431b25205c2c8dc09fda861e51e0e" @@ -6176,11 +6110,6 @@ for-each@^0.3.3: dependencies: is-callable "^1.1.3" -forever-agent@~0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" - integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE= - form-data@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" @@ -6190,15 +6119,6 @@ form-data@^4.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" -form-data@~2.3.2: - version "2.3.3" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" - integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== - dependencies: - asynckit "^0.4.0" - combined-stream "^1.0.6" - mime-types "^2.1.12" - format@^0.2.0: version "0.2.2" resolved "https://registry.yarnpkg.com/format/-/format-0.2.2.tgz#d6170107e9efdc4ed30c9dc39016df942b5cb58b" @@ -6295,13 +6215,6 @@ get-symbol-description@^1.0.0: call-bind "^1.0.2" get-intrinsic "^1.1.1" -getpass@^0.1.1: - version "0.1.7" - resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" - integrity sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo= - dependencies: - assert-plus "^1.0.0" - glob-parent@^5.1.2, glob-parent@~5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" @@ -6438,19 +6351,6 @@ gzip-size@^6.0.0: dependencies: duplexer "^0.1.2" -har-schema@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" - integrity sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI= - -har-validator@~5.1.3: - version "5.1.5" - resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.5.tgz#1f0803b9f8cb20c0fa13822df1ecddb36bde1efd" - integrity sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w== - dependencies: - ajv "^6.12.3" - har-schema "^2.0.0" - has-bigints@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.2.tgz#0871bd3e3d51626f6ca0966668ba35d5602d6eaa" @@ -6672,15 +6572,6 @@ http-proxy-agent@^7.0.0: agent-base "^7.1.0" debug "^4.3.4" -http-signature@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" - integrity sha1-muzZJRFHcvPZW2WmCruPfBj7rOE= - dependencies: - assert-plus "^1.0.0" - jsprim "^1.2.2" - sshpk "^1.7.0" - https-proxy-agent@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" @@ -7094,11 +6985,6 @@ is-typed-array@^1.1.10: dependencies: which-typed-array "^1.1.11" -is-typedarray@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" - integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= - is-unc-path@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-unc-path/-/is-unc-path-1.0.0.tgz#d731e8898ed090a12c352ad2eaed5095ad322c9d" @@ -7185,11 +7071,6 @@ isomorphic.js@^0.2.4: resolved "https://registry.yarnpkg.com/isomorphic.js/-/isomorphic.js-0.2.5.tgz#13eecf36f2dba53e85d355e11bf9d4208c6f7f88" integrity sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw== -isstream@~0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" - integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo= - jake@^10.8.5: version "10.8.7" resolved "https://registry.yarnpkg.com/jake/-/jake-10.8.7.tgz#63a32821177940c33f356e0ba44ff9d34e1c7d8f" @@ -7232,11 +7113,6 @@ js-yaml@^4.0.0, js-yaml@^4.1.0: dependencies: argparse "^2.0.1" -jsbn@~0.1.0: - version "0.1.1" - resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" - integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM= - jsdom@^22.1.0: version "22.1.0" resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-22.1.0.tgz#0fca6d1a37fbeb7f4aac93d1090d782c56b611c8" @@ -7281,7 +7157,7 @@ json-schema-traverse@^0.4.1: resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== -json-schema@0.2.3, json-schema@0.4.0: +json-schema@0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5" integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA== @@ -7298,11 +7174,6 @@ json-stable-stringify@^1.0.1: dependencies: jsonify "~0.0.0" -json-stringify-safe@~5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" - integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus= - json-to-pretty-yaml@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/json-to-pretty-yaml/-/json-to-pretty-yaml-1.2.2.tgz#f4cd0bd0a5e8fe1df25aaf5ba118b099fd992d5b" @@ -7349,16 +7220,6 @@ jsonify@~0.0.0: resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" integrity sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM= -jsprim@^1.2.2: - version "1.4.1" - resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" - integrity sha1-MT5mvB5cwG5Di8G3SZwuXFastqI= - dependencies: - assert-plus "1.0.0" - extsprintf "1.3.0" - json-schema "0.2.3" - verror "1.10.0" - "jsx-ast-utils@^2.4.1 || ^3.0.0", jsx-ast-utils@^3.3.3: version "3.3.3" resolved "https://registry.yarnpkg.com/jsx-ast-utils/-/jsx-ast-utils-3.3.3.tgz#76b3e6e6cece5c69d49a5792c3d01bd1a0cdc7ea" @@ -7471,11 +7332,6 @@ lodash.merge@^4.6.2: resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== -lodash.pick@4.4.0: - version "4.4.0" - resolved "https://registry.npmjs.org/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3" - integrity sha1-UvBWEP/53tQiYRRB7R/BI6AwAbM= - lodash@^4.0.1, lodash@^4.17.11, lodash@^4.17.15, lodash@^4.17.20, lodash@^4.17.21, lodash@~4.17.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" @@ -7810,7 +7666,7 @@ mime-db@1.52.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== -mime-types@^2.1.12, mime-types@~2.1.19: +mime-types@^2.1.12: version "2.1.35" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== @@ -8027,11 +7883,6 @@ nwsapi@^2.2.4: resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.7.tgz#738e0707d3128cb750dddcfe90e4610482df0f30" integrity sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ== -oauth-sign@~0.9.0: - version "0.9.0" - resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" - integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== - object-assign@^4.1.0, object-assign@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" @@ -8371,11 +8222,6 @@ pathval@^1.1.1: resolved "https://registry.yarnpkg.com/pathval/-/pathval-1.1.1.tgz#8534e77a77ce7ac5a2512ea21e0fdb8fcf6c3d8d" integrity sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ== -performance-now@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" - integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= - picocolors@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" @@ -8654,11 +8500,6 @@ prr@~1.0.1: resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" integrity sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw== -psl@^1.1.28: - version "1.8.0" - resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" - integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== - psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" @@ -8691,11 +8532,6 @@ pvutils@^1.1.3: resolved "https://registry.yarnpkg.com/pvutils/-/pvutils-1.1.3.tgz#f35fc1d27e7cd3dfbd39c0826d173e806a03f5a3" integrity sha512-pMpnA0qRdFp32b1sJl1wOJNxZLQ2cbQx+k6tjNtZ8CpvVhNqEPRgivZ2WOUev2YMajecdH7ctUPDvEe87nariQ== -qs@~6.5.2: - version "6.5.3" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.3.tgz#3aeeffc91967ef6e35c0e488ef46fb296ab76aad" - integrity sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA== - query-string@*: version "7.0.0" resolved "https://registry.yarnpkg.com/query-string/-/query-string-7.0.0.tgz#aaad2c8d5c6a6d0c6afada877fecbd56af79e609" @@ -9322,16 +9158,15 @@ reactcss@^1.2.0: dependencies: lodash "^4.0.1" -reactour@1.18.7: - version "1.18.7" - resolved "https://registry.yarnpkg.com/reactour/-/reactour-1.18.7.tgz#86a61869890bc3444c2bf412d30833eb85352525" - integrity sha512-kkXy4h5+fieNPzrPYdWiLj6afl+xH2NQw4En9XJD9EwCIGsRmfcppyJ1xwqJDC6JRhPttf+5wUIsUoLSOlk/Ag== +reactour@^1.19.3: + version "1.19.3" + resolved "https://registry.yarnpkg.com/reactour/-/reactour-1.19.3.tgz#d83877496a3af211a881fba3709276accf2ac3e7" + integrity sha512-a2/j+xmbGlPMQhMy/zdpSmJN1STc0PcLUxhJbAC+bOcYdWHStFR02j6dhNuF/+IeqpX/d44PvlrRXSTt4LGk6w== dependencies: "@rooks/use-mutation-observer" "4.11.2" classnames "2.3.1" focus-outline-manager "^1.0.2" lodash.debounce "4.0.8" - lodash.pick "4.4.0" prop-types "15.7.2" react-focus-lock "2.5.2" scroll-smooth "1.1.1" @@ -9569,32 +9404,6 @@ repeat-string@^1.0.0: resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc= -request@^2.88.2: - version "2.88.2" - resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" - integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== - dependencies: - aws-sign2 "~0.7.0" - aws4 "^1.8.0" - caseless "~0.12.0" - combined-stream "~1.0.6" - extend "~3.0.2" - forever-agent "~0.6.1" - form-data "~2.3.2" - har-validator "~5.1.3" - http-signature "~1.2.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.19" - oauth-sign "~0.9.0" - performance-now "^2.1.0" - qs "~6.5.2" - safe-buffer "^5.1.2" - tough-cookie "~2.5.0" - tunnel-agent "^0.6.0" - uuid "^3.3.2" - require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -9744,7 +9553,7 @@ safari-14-idb-fix@^1.0.6: resolved "https://registry.yarnpkg.com/safari-14-idb-fix/-/safari-14-idb-fix-1.0.6.tgz#cbaabc33a4500c44b5c432d6c525b0ed9b68bb65" integrity sha512-oTEQOdMwRX+uCtWCKT1nx2gAeSdpr8elg/2gcaKUH00SJU2xWESfkx11nmXwTRHy7xfQoj1o4TTQvdmuBosTnA== -safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0: +safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== @@ -9758,7 +9567,7 @@ safe-regex-test@^1.0.0: get-intrinsic "^1.1.3" is-regex "^1.1.4" -"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: +"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0": version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== @@ -10029,21 +9838,6 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= -sshpk@^1.7.0: - version "1.16.1" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877" - integrity sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg== - dependencies: - asn1 "~0.2.3" - assert-plus "^1.0.0" - bcrypt-pbkdf "^1.0.0" - dashdash "^1.12.0" - ecc-jsbn "~0.1.1" - getpass "^0.1.1" - jsbn "~0.1.0" - safer-buffer "^2.0.2" - tweetnacl "~0.14.0" - stack-generator@^2.0.5: version "2.0.10" resolved "https://registry.yarnpkg.com/stack-generator/-/stack-generator-2.0.10.tgz#8ae171e985ed62287d4f1ed55a1633b3fb53bb4d" @@ -10387,14 +10181,6 @@ tough-cookie@^4.1.2: universalify "^0.2.0" url-parse "^1.5.3" -tough-cookie@~2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" - integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== - dependencies: - psl "^1.1.28" - punycode "^2.1.1" - tr46@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/tr46/-/tr46-4.1.1.tgz#281a758dcc82aeb4fe38c7dfe4d11a395aac8469" @@ -10486,13 +10272,6 @@ tsutils@^3.21.0: dependencies: tslib "^1.8.1" -tunnel-agent@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" - integrity sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0= - dependencies: - safe-buffer "^5.0.1" - turndown-plugin-gfm@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.2.tgz#6f8678a361f35220b2bdf5619e6049add75bf1c7" @@ -10505,11 +10284,6 @@ turndown@^7.1.1: dependencies: domino "^2.1.6" -tweetnacl@^0.14.3, tweetnacl@~0.14.0: - version "0.14.5" - resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" - integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q= - type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" @@ -10633,15 +10407,6 @@ unist-util-visit@^2.0.0, unist-util-visit@^2.0.3: unist-util-is "^4.0.0" unist-util-visit-parents "^3.0.0" -universal-analytics@^0.4.20: - version "0.4.23" - resolved "https://registry.yarnpkg.com/universal-analytics/-/universal-analytics-0.4.23.tgz#d915e676850c25c4156762471bdd7cf2eaaca8ac" - integrity sha512-lgMIH7XBI6OgYn1woDEmxhGdj8yDefMKg7GkWdeATAlQZFrMrNyxSkpDzY57iY0/6fdlzTbBV03OawvvzG+q7A== - dependencies: - debug "^4.1.1" - request "^2.88.2" - uuid "^3.0.0" - universalify@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.2.0.tgz#6451760566fa857534745ab1dde952d1b1761be0" @@ -10746,11 +10511,6 @@ util-deprecate@^1.0.1: resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= -uuid@^3.0.0, uuid@^3.3.2: - version "3.4.0" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" - integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== - uuid@^8.3.2: version "8.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" @@ -10766,15 +10526,6 @@ value-or-promise@^1.0.11, value-or-promise@^1.0.12: resolved "https://registry.yarnpkg.com/value-or-promise/-/value-or-promise-1.0.12.tgz#0e5abfeec70148c78460a849f6b003ea7986f15c" integrity sha512-Z6Uz+TYwEqE7ZN50gwn+1LCVo9ZVrpxRPOhOLnncYkY1ZzOYtrX8Fwf/rFktZ8R5mJms6EZf5TqNOMeZmnPq9Q== -verror@1.10.0: - version "1.10.0" - resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" - integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA= - dependencies: - assert-plus "^1.0.0" - core-util-is "1.0.2" - extsprintf "^1.2.0" - vfile-location@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/vfile-location/-/vfile-location-3.2.0.tgz#d8e41fbcbd406063669ebf6c33d56ae8721d0f3c" @@ -10845,10 +10596,10 @@ vite-plugin-svgr@^4.1.0: "@svgr/core" "^8.1.0" "@svgr/plugin-jsx" "^8.1.0" -"vite@^3.0.0 || ^4.0.0 || ^5.0.0-0", "vite@^3.1.0 || ^4.0.0 || ^5.0.0-0", vite@^4.5.2: - version "4.5.2" - resolved "https://registry.yarnpkg.com/vite/-/vite-4.5.2.tgz#d6ea8610e099851dad8c7371599969e0f8b97e82" - integrity sha512-tBCZBNSBbHQkaGyhGCDUGqeo2ph8Fstyp6FMSvTtsXeZSPpSMGlviAOav2hxVTqFcx8Hj/twtWKsMJXNY0xI8w== +"vite@^3.0.0 || ^4.0.0 || ^5.0.0-0", "vite@^3.1.0 || ^4.0.0 || ^5.0.0-0", vite@^4.5.3: + version "4.5.3" + resolved "https://registry.yarnpkg.com/vite/-/vite-4.5.3.tgz#d88a4529ea58bae97294c7e2e6f0eab39a50fb1a" + integrity sha512-kQL23kMeX92v3ph7IauVkXkikdDRsYMGTVl5KY2E9OY4ONLvkHf04MDTbnfo6NKxZiDLWzVpP5oTa8hQD8U3dg== dependencies: esbuild "^0.18.10" postcss "^8.4.27" diff --git a/docker/datahub-frontend/Dockerfile b/docker/datahub-frontend/Dockerfile index 5563fd6350e20..a828f1d8c27ad 100644 --- a/docker/datahub-frontend/Dockerfile +++ b/docker/datahub-frontend/Dockerfile @@ -1,7 +1,7 @@ # Defining environment ARG APP_ENV=prod -FROM alpine:3.18 AS base +FROM alpine:3.20 AS base # Configurable repositories ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine @@ -17,7 +17,7 @@ RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762 ENV JMX_VERSION=0.18.0 RUN apk --no-cache --update-cache --available upgrade \ - && apk --no-cache add curl sqlite libc6-compat java-snappy \ + && apk --no-cache add curl sqlite libc6-compat snappy \ && apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \ && apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \ && wget ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar -O opentelemetry-javaagent.jar \ diff --git a/docker/datahub-gms/Dockerfile b/docker/datahub-gms/Dockerfile index a9399a24f3fbf..d30dbd8493057 100644 --- a/docker/datahub-gms/Dockerfile +++ b/docker/datahub-gms/Dockerfile @@ -6,8 +6,7 @@ ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine ARG GITHUB_REPO_URL=https://github.com ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2 -FROM golang:1-alpine3.18 AS binary -FROM golang:1-alpine3.18 AS binary +FROM golang:1-alpine3.20 AS binary # Re-declaring arg from above to make it available in this stage (will inherit default value) ARG ALPINE_REPO_URL @@ -24,27 +23,28 @@ WORKDIR /go/src/github.com/jwilder/dockerize RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION -FROM alpine:3.18 AS base +FROM alpine:3.20 AS base -# Upgrade Alpine and base packages ENV JMX_VERSION=0.18.0 +ENV JETTY_VERSION=11.0.21 # Re-declaring args from above to make them available in this stage (will inherit default values) ARG ALPINE_REPO_URL ARG GITHUB_REPO_URL ARG MAVEN_CENTRAL_REPO_URL +# Upgrade Alpine and base packages # Optionally set corporate mirror for apk RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762 RUN apk --no-cache --update-cache --available upgrade \ - && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat java-snappy \ + && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat snappy \ && apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \ && apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \ - && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-runner/11.0.19/jetty-runner-11.0.19.jar --output jetty-runner.jar \ - && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-jmx/11.0.19/jetty-jmx-11.0.19.jar --output jetty-jmx.jar \ - && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-util/11.0.19/jetty-util-11.0.19.jar --output jetty-util.jar \ + && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-runner/${JETTY_VERSION}/jetty-runner-${JETTY_VERSION}.jar --output jetty-runner.jar \ + && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-jmx/${JETTY_VERSION}/jetty-jmx-${JETTY_VERSION}.jar --output jetty-jmx.jar \ + && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-util/${JETTY_VERSION}/jetty-util-${JETTY_VERSION}.jar --output jetty-util.jar \ && wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \ && wget --no-verbose ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \ && cp /usr/lib/jvm/java-17-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks diff --git a/docker/datahub-gms/jetty.xml b/docker/datahub-gms/jetty.xml index 589442b115bfc..3f04635d9498c 100644 --- a/docker/datahub-gms/jetty.xml +++ b/docker/datahub-gms/jetty.xml @@ -31,7 +31,7 @@ - + diff --git a/docker/datahub-ingestion-base/Dockerfile b/docker/datahub-ingestion-base/Dockerfile index b7d5a11fdf6d8..383478b675640 100644 --- a/docker/datahub-ingestion-base/Dockerfile +++ b/docker/datahub-ingestion-base/Dockerfile @@ -7,7 +7,7 @@ ARG GITHUB_REPO_URL=https://github.com ARG DEBIAN_REPO_URL=https://deb.debian.org/debian ARG PIP_MIRROR_URL=https://pypi.python.org/simple -FROM golang:1-alpine3.18 AS dockerize-binary +FROM golang:1-alpine3.20 AS dockerize-binary # Re-declaring arg from above to make it available in this stage (will inherit default value) ARG ALPINE_REPO_URL diff --git a/docker/datahub-ingestion-base/base-requirements.txt b/docker/datahub-ingestion-base/base-requirements.txt index 82be7ce1a6e66..2f2b64723f0ed 100644 --- a/docker/datahub-ingestion-base/base-requirements.txt +++ b/docker/datahub-ingestion-base/base-requirements.txt @@ -1,25 +1,25 @@ # Generated requirements file. Run ./regenerate-base-requirements.sh to regenerate. acryl-datahub-classify==0.0.10 acryl-PyHive==0.6.16 -acryl-sqlglot==23.11.2.dev2 +acryl-sqlglot==24.0.1.dev7 aenum==3.1.15 aiohttp==3.9.5 aiosignal==1.3.1 alembic==1.13.1 altair==4.2.0 -anyio==4.3.0 -apache-airflow==2.9.0 -apache-airflow-providers-common-io==1.3.1 -apache-airflow-providers-common-sql==1.12.0 -apache-airflow-providers-fab==1.0.4 -apache-airflow-providers-ftp==3.8.0 -apache-airflow-providers-http==4.10.1 -apache-airflow-providers-imap==3.5.0 -apache-airflow-providers-smtp==1.6.1 -apache-airflow-providers-sqlite==3.7.1 +anyio==4.4.0 +apache-airflow==2.9.2 +apache-airflow-providers-common-io==1.3.2 +apache-airflow-providers-common-sql==1.14.0 +apache-airflow-providers-fab==1.1.1 +apache-airflow-providers-ftp==3.9.1 +apache-airflow-providers-http==4.11.1 +apache-airflow-providers-imap==3.6.1 +apache-airflow-providers-smtp==1.7.1 +apache-airflow-providers-sqlite==3.8.1 apispec==6.6.1 appnope==0.1.4 -argcomplete==3.3.0 +argcomplete==3.4.0 argon2-cffi==23.1.0 argon2-cffi-bindings==21.2.0 asgiref==3.8.1 @@ -29,22 +29,22 @@ async-timeout==4.0.3 asynch==0.2.3 attrs==23.2.0 avro==1.11.3 -avro-gen3==0.7.12 -Babel==2.14.0 +avro-gen3==0.7.13 +Babel==2.15.0 backoff==2.2.1 beautifulsoup4==4.12.3 bleach==6.1.0 -blinker==1.7.0 +blinker==1.8.2 blis==0.7.11 -boto3==1.34.90 -botocore==1.34.90 +boto3==1.34.129 +botocore==1.34.129 bracex==2.4 cached-property==1.5.2 cachelib==0.9.0 cachetools==5.3.3 catalogue==2.0.10 cattrs==23.2.3 -certifi==2024.2.2 +certifi==2024.6.2 cffi==1.16.0 chardet==5.2.0 charset-normalizer==3.3.2 @@ -53,72 +53,73 @@ click==8.1.7 click-default-group==1.2.4 click-spinner==0.1.10 clickclick==20.10.2 -clickhouse-driver==0.2.7 +clickhouse-driver==0.2.8 clickhouse-sqlalchemy==0.2.4 cloudpickle==3.0.0 colorama==0.4.6 colorlog==4.8.0 comm==0.2.2 -confection==0.1.4 +confection==0.1.5 ConfigUpdater==3.2 -confluent-kafka==2.3.0 -connexion==2.14.2 +confluent-kafka==2.4.0 +connexion==2.14.1 cron-descriptor==1.4.3 croniter==2.0.5 -cryptography==42.0.5 +cryptography==42.0.8 cx_Oracle==8.3.0 cymem==2.0.8 databricks-dbapi==0.6.0 -databricks-sdk==0.25.1 +databricks-sdk==0.28.0 databricks-sql-connector==2.9.6 dataflows-tabulator==1.54.3 db-dtypes==1.2.0 debugpy==1.8.1 decorator==5.1.1 defusedxml==0.7.1 -deltalake==0.17.1 +deltalake==0.17.4 Deprecated==1.2.14 dill==0.3.8 dnspython==2.6.1 -docker==7.0.0 +docker==7.1.0 docutils==0.21.2 ecdsa==0.19.0 elasticsearch==7.13.4 -email_validator==2.1.1 +email_validator==2.1.2 entrypoints==0.4 et-xmlfile==1.1.0 exceptiongroup==1.2.1 executing==2.0.1 expandvars==0.12.0 fastavro==1.9.4 -fastjsonschema==2.19.1 -filelock==3.13.4 +fastjsonschema==2.20.0 +filelock==3.15.1 Flask==2.2.5 flatdict==4.0.1 frozenlist==1.4.1 fsspec==2023.12.2 future==1.0.0 -GeoAlchemy2==0.15.0 +GeoAlchemy2==0.15.1 gitdb==4.0.11 GitPython==3.1.43 -google-api-core==2.18.0 -google-auth==2.29.0 +google-api-core==2.19.0 +google-auth==2.30.0 google-cloud-appengine-logging==1.4.3 google-cloud-audit-log==0.2.5 -google-cloud-bigquery==3.21.0 +google-cloud-bigquery==3.24.0 google-cloud-core==2.4.1 +google-cloud-datacatalog==3.19.0 google-cloud-datacatalog-lineage==0.2.2 google-cloud-logging==3.5.0 google-crc32c==1.5.0 -google-re2==1.1 -google-resumable-media==2.7.0 -googleapis-common-protos==1.63.0 +google-re2==1.1.20240601 +google-resumable-media==2.7.1 +googleapis-common-protos==1.63.1 gql==3.5.0 graphql-core==3.2.3 great-expectations==0.15.50 greenlet==3.0.3 grpc-google-iam-v1==0.13.0 -grpcio==1.62.2 +grpcio==1.64.1 grpcio-status==1.62.2 grpcio-tools==1.62.2 gssapi==1.8.3 @@ -128,96 +129,96 @@ httpcore==1.0.5 httpx==0.27.0 humanfriendly==10.0 idna==3.7 -ijson==3.2.3 -importlib-metadata==7.0.0 +ijson==3.3.0 +importlib_metadata==7.1.0 importlib_resources==6.4.0 inflection==0.5.1 ipaddress==1.0.23 ipykernel==6.17.1 ipython==8.21.0 ipython-genutils==0.2.0 -ipywidgets==8.1.2 -iso3166==2.1.1 +ipywidgets==8.1.3 isodate==0.6.1 itsdangerous==2.2.0 jedi==0.19.1 -Jinja2==3.1.3 +Jinja2==3.1.4 jmespath==1.0.1 JPype1==1.5.0 jsonlines==4.0.0 jsonpatch==1.33 -jsonpointer==2.4 +jsonpointer==3.0.0 jsonref==1.1.0 -jsonschema==4.21.1 +jsonschema==4.22.0 jsonschema-specifications==2023.12.1 jupyter-server==1.16.0 jupyter_client==7.4.9 jupyter_core==4.12.0 jupyterlab_pygments==0.3.0 -jupyterlab_widgets==3.0.10 +jupyterlab_widgets==3.0.11 langcodes==3.4.0 language_data==1.2.0 lark==1.1.4 lazy-object-proxy==1.10.0 leb128==1.0.7 -limits==3.11.0 +limits==3.12.0 linear-tsv==1.1.0 linkify-it-py==2.0.3 lkml==1.3.4 lockfile==0.12.2 looker-sdk==23.0.0 -lxml==5.2.1 +lxml==5.2.2 lz4==4.3.3 makefun==1.15.2 -Mako==1.3.3 -marisa-trie==1.1.0 +Mako==1.3.5 +marisa-trie==1.2.0 markdown-it-py==3.0.0 MarkupSafe==2.1.5 -marshmallow==3.21.1 +marshmallow==3.21.3 marshmallow-oneofschema==3.1.1 marshmallow-sqlalchemy==0.28.2 matplotlib-inline==0.1.7 -mdit-py-plugins==0.4.0 +mdit-py-plugins==0.4.1 mdurl==0.1.2 +methodtools==0.4.7 mistune==3.0.2 mixpanel==4.10.1 -mlflow-skinny==2.12.1 +mlflow-skinny==2.14.0 mmhash3==3.0.1 -more-itertools==10.2.0 +more-itertools==10.3.0 moto==4.2.14 msal==1.22.0 multidict==6.0.5 murmurhash==1.0.10 mypy-extensions==1.0.0 -nbclassic==1.0.0 +nbclassic==1.1.0 nbclient==0.6.3 -nbconvert==7.16.3 +nbconvert==7.16.4 nbformat==5.10.4 nest-asyncio==1.6.0 networkx==3.3 -notebook==6.5.6 +notebook==6.5.7 notebook_shim==0.2.4 numpy==1.26.4 oauthlib==3.2.2 okta==1.7.0 -openlineage-airflow==1.7.0 -openlineage-integration-common==1.7.0 -openlineage-python==1.7.0 -openlineage_sql==1.7.0 -openpyxl==3.1.2 -opentelemetry-api==1.24.0 -opentelemetry-exporter-otlp==1.24.0 -opentelemetry-exporter-otlp-proto-common==1.24.0 -opentelemetry-exporter-otlp-proto-grpc==1.24.0 -opentelemetry-exporter-otlp-proto-http==1.24.0 -opentelemetry-proto==1.24.0 -opentelemetry-sdk==1.24.0 -opentelemetry-semantic-conventions==0.45b0 +openlineage-airflow==1.12.0 +openlineage-integration-common==1.12.0 +openlineage-python==1.12.0 +openlineage_sql==1.12.0 +openpyxl==3.1.4 +opentelemetry-api==1.25.0 +opentelemetry-exporter-otlp==1.25.0 +opentelemetry-exporter-otlp-proto-common==1.25.0 +opentelemetry-exporter-otlp-proto-grpc==1.25.0 +opentelemetry-exporter-otlp-proto-http==1.25.0 +opentelemetry-proto==1.25.0 +opentelemetry-sdk==1.25.0 +opentelemetry-semantic-conventions==0.46b0 ordered-set==4.1.0 -packaging==24.0 +packaging==24.1 pandas==2.1.4 pandocfilters==1.5.1 -parse==1.20.1 +parse==1.20.2 parso==0.8.4 pathlib_abc==0.1.1 pathspec==0.12.1 @@ -225,43 +226,44 @@ pathy==0.11.0 pendulum==3.0.0 pexpect==4.9.0 phonenumbers==8.13.0 -platformdirs==4.2.1 +platformdirs==4.2.2 pluggy==1.5.0 preshed==3.0.9 prison==0.2.1 progressbar2==4.4.2 prometheus_client==0.20.0 -prompt-toolkit==3.0.43 +prompt_toolkit==3.0.47 proto-plus==1.23.0 protobuf==4.25.3 -psutil==5.9.8 +psutil==6.0.0 psycopg2-binary==2.9.9 ptyprocess==0.7.0 pure-eval==0.2.2 pure-sasl==0.6.2 py-partiql-parser==0.5.0 -pyarrow==16.0.0 +pyarrow==16.1.0 pyarrow-hotfix==0.6 pyasn1==0.6.0 pyasn1_modules==0.4.0 pyathena==2.25.2 -pycountry==23.12.11 +pycountry==24.6.1 pycparser==2.22 pycryptodome==3.20.0 -pydantic==1.10.15 -pydash==8.0.0 -pydruid==0.6.6 -Pygments==2.17.2 +pydantic==1.10.16 +pydash==8.0.1 +pydruid==0.6.9 +Pygments==2.18.0 pyiceberg==0.4.0 -pymongo==4.6.3 -PyMySQL==1.1.0 +pymongo==4.7.3 +PyMySQL==1.1.1 pyOpenSSL==24.1.0 pyparsing==3.0.9 -pyspnego==0.10.2 +pyspnego==0.11.0 python-daemon==3.0.1 python-dateutil==2.9.0.post0 python-jose==3.3.0 python-ldap==3.4.4 +python-liquid==1.12.1 python-nvd3==0.16.0 python-slugify==8.0.4 python-stdnum==1.20 @@ -269,30 +271,31 @@ python-tds==1.15.0 python-utils==3.8.2 pytz==2024.1 PyYAML==6.0.1 -pyzmq==24.0.1 +pyzmq==26.0.3 redash-toolbelt==0.1.9 redshift-connector==2.1.1 -referencing==0.35.0 -regex==2024.4.16 -requests==2.31.0 -requests-file==2.0.0 +referencing==0.35.1 +regex==2024.5.15 +requests==2.32.3 +requests-file==2.1.0 requests-gssapi==1.3.0 -requests-ntlm==1.2.0 requests-toolbelt==1.0.0 -responses==0.25.0 +requests_ntlm==1.3.0 +responses==0.25.3 rfc3339-validator==0.1.4 rfc3986==2.0.0 rich==13.7.1 -rich-argparse==1.4.0 -rpds-py==0.18.0 +rich-argparse==1.5.2 +rpds-py==0.18.1 rsa==4.9 +rstr==3.2.2 ruamel.yaml==0.17.17 s3transfer==0.10.1 -schwifty==2024.4.0 -scipy==1.13.0 +schwifty==2024.6.1 +scipy==1.13.1 scramp==1.4.5 Send2Trash==1.8.3 -sentry-sdk==1.45.0 +sentry-sdk==2.5.1 setproctitle==1.3.3 simple-salesforce==1.12.6 six==1.16.0 @@ -300,7 +303,7 @@ slack-sdk==3.18.1 smart-open==6.4.0 smmap==5.0.1 sniffio==1.3.1 -snowflake-connector-python==3.9.1 +snowflake-connector-python==3.11.0 snowflake-sqlalchemy==1.5.3 sortedcontainers==2.4.0 soupsieve==2.5 @@ -315,6 +318,7 @@ SQLAlchemy-JSONField==1.0.2 sqlalchemy-pytds==0.3.5 sqlalchemy-redshift==0.8.14 SQLAlchemy-Utils==0.41.2 +sqlglotrs==0.2.5 sqllineage==1.3.8 sqlparse==0.4.4 srsly==2.4.8 @@ -323,9 +327,9 @@ strictyaml==1.7.3 tableauserverclient==0.25 tableschema==1.20.11 tabulate==0.9.0 -tenacity==8.2.3 -teradatasql==20.0.0.10 -teradatasqlalchemy==20.0.0.0 +tenacity==8.4.1 +teradatasql==20.0.0.12 +teradatasqlalchemy==20.0.0.1 termcolor==2.4.0 terminado==0.18.1 text-unidecode==1.3 @@ -335,37 +339,38 @@ thrift-sasl==0.4.3 time-machine==2.14.1 tinycss2==1.3.0 toml==0.10.2 -tomlkit==0.12.4 +tomlkit==0.12.5 toolz==0.12.1 -tornado==6.4 -tqdm==4.66.2 +tornado==6.4.1 +tqdm==4.66.4 traitlets==5.2.1.post0 trino==0.328.0 typer==0.7.0 typing-inspect==0.9.0 -typing_extensions==4.11.0 +typing_extensions==4.12.2 tzdata==2024.1 tzlocal==5.2 uc-micro-py==1.0.3 -ujson==5.9.0 +ujson==5.10.0 unicodecsv==0.14.1 universal_pathlib==0.2.2 -urllib3==1.26.18 +urllib3==1.26.19 vertica-python==1.3.8 -vertica-sqlalchemy-dialect==0.0.8.1 +vertica-sqlalchemy-dialect==0.0.8.2 vininfo==1.8.0 -wasabi==1.1.2 -wcmatch==8.5.1 +wasabi==1.1.3 +wcmatch==8.5.2 wcwidth==0.2.13 webencodings==0.5.1 websocket-client==1.8.0 -Werkzeug==2.2.3 -widgetsnbextension==4.0.10 +Werkzeug==2.3.8 +widgetsnbextension==4.0.11 +wirerope==0.4.7 wrapt==1.16.0 WTForms==3.1.2 xlrd==2.0.1 xmltodict==0.13.0 yarl==1.9.4 zeep==4.2.1 -zipp==3.18.1 +zipp==3.19.2 zstd==1.5.5.1 diff --git a/docker/datahub-mae-consumer/Dockerfile b/docker/datahub-mae-consumer/Dockerfile index 9b7c6e762462e..0ee55821f2579 100644 --- a/docker/datahub-mae-consumer/Dockerfile +++ b/docker/datahub-mae-consumer/Dockerfile @@ -6,7 +6,7 @@ ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine ARG GITHUB_REPO_URL=https://github.com ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2 -FROM golang:1-alpine3.18 AS binary +FROM golang:1-alpine3.20 AS binary # Re-declaring arg from above to make it available in this stage (will inherit default value) ARG ALPINE_REPO_URL @@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION -FROM alpine:3.18 AS base +FROM alpine:3.20 AS base # Re-declaring args from above to make them available in this stage (will inherit default values) ARG ALPINE_REPO_URL @@ -37,7 +37,7 @@ RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then ENV JMX_VERSION=0.18.0 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762 RUN apk --no-cache --update-cache --available upgrade \ - && apk --no-cache add curl bash coreutils sqlite libc6-compat java-snappy \ + && apk --no-cache add curl bash coreutils sqlite libc6-compat snappy \ && apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \ && apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \ && wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \ diff --git a/docker/datahub-mce-consumer/Dockerfile b/docker/datahub-mce-consumer/Dockerfile index 4da94794e0ead..8f85b432a1071 100644 --- a/docker/datahub-mce-consumer/Dockerfile +++ b/docker/datahub-mce-consumer/Dockerfile @@ -6,7 +6,7 @@ ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine ARG GITHUB_REPO_URL=https://github.com ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2 -FROM golang:1-alpine3.18 AS binary +FROM golang:1-alpine3.20 AS binary # Re-declaring arg from above to make it available in this stage (will inherit default value) ARG ALPINE_REPO_URL @@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION -FROM alpine:3.18 AS base +FROM alpine:3.20 AS base # Re-declaring args from above to make them available in this stage (will inherit default values) ARG ALPINE_REPO_URL @@ -37,7 +37,7 @@ RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then ENV JMX_VERSION=0.18.0 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762 RUN apk --no-cache --update-cache --available upgrade \ - && apk --no-cache add curl bash sqlite libc6-compat java-snappy \ + && apk --no-cache add curl bash sqlite libc6-compat snappy \ && apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \ && apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \ && wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \ diff --git a/docker/datahub-upgrade/Dockerfile b/docker/datahub-upgrade/Dockerfile index cda13378be68e..675e24ab87109 100644 --- a/docker/datahub-upgrade/Dockerfile +++ b/docker/datahub-upgrade/Dockerfile @@ -6,7 +6,7 @@ ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine ARG GITHUB_REPO_URL=https://github.com ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2 -FROM golang:1-alpine3.18 AS binary +FROM golang:1-alpine3.20 AS binary # Re-declaring arg from above to make it available in this stage (will inherit default value) ARG ALPINE_REPO_URL @@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION -FROM alpine:3.18 AS base +FROM alpine:3.20 AS base # Re-declaring args from above to make them available in this stage (will inherit default values) ARG ALPINE_REPO_URL @@ -33,15 +33,17 @@ ARG MAVEN_CENTRAL_REPO_URL # Optionally set corporate mirror for apk RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi -# Upgrade Alpine and base packages ENV JMX_VERSION=0.18.0 +ENV JETTY_VERSION=11.0.21 + +# Upgrade Alpine and base packages # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762 RUN apk --no-cache --update-cache --available upgrade \ - && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat java-snappy \ + && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat snappy \ && apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \ - && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-runner/11.0.19/jetty-runner-11.0.19.jar --output jetty-runner.jar \ - && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-jmx/11.0.19/jetty-jmx-11.0.19.jar --output jetty-jmx.jar \ - && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-util/11.0.19/jetty-util-11.0.19.jar --output jetty-util.jar \ + && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-runner/${JETTY_VERSION}/jetty-runner-${JETTY_VERSION}.jar --output jetty-runner.jar \ + && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-jmx/${JETTY_VERSION}/jetty-jmx-${JETTY_VERSION}.jar --output jetty-jmx.jar \ + && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-util/${JETTY_VERSION}/jetty-util-${JETTY_VERSION}.jar --output jetty-util.jar \ && wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \ && wget --no-verbose ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \ && cp /usr/lib/jvm/java-17-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks diff --git a/docker/elasticsearch-setup/Dockerfile b/docker/elasticsearch-setup/Dockerfile index fdaf9ddbaf813..7390e3579dcf8 100644 --- a/docker/elasticsearch-setup/Dockerfile +++ b/docker/elasticsearch-setup/Dockerfile @@ -6,7 +6,7 @@ ARG APP_ENV=prod # Defining custom repo urls for use in enterprise environments. Re-used between stages below. ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine -FROM golang:1-alpine3.18 AS binary +FROM golang:1-alpine3.20 AS binary ARG ALPINE_REPO_URL @@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION -FROM alpine:3.18 AS base +FROM alpine:3.20 AS base ARG ALPINE_REPO_URL diff --git a/docker/mysql-setup/Dockerfile b/docker/mysql-setup/Dockerfile index 409f96a325830..46969352d8174 100644 --- a/docker/mysql-setup/Dockerfile +++ b/docker/mysql-setup/Dockerfile @@ -1,7 +1,7 @@ # Defining custom repo urls for use in enterprise environments. Re-used between stages below. ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine -FROM golang:1-alpine3.18 AS binary +FROM golang:1-alpine3.20 AS binary ARG ALPINE_REPO_URL @@ -17,7 +17,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION -FROM alpine:3.18 +FROM alpine:3.20 COPY --from=binary /go/bin/dockerize /usr/local/bin ARG ALPINE_REPO_URL diff --git a/docker/postgres-setup/Dockerfile b/docker/postgres-setup/Dockerfile index 673ce979477be..8ab211218f240 100644 --- a/docker/postgres-setup/Dockerfile +++ b/docker/postgres-setup/Dockerfile @@ -1,7 +1,7 @@ # Defining custom repo urls for use in enterprise environments. Re-used between stages below. ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine -FROM golang:1-alpine3.18 AS binary +FROM golang:1-alpine3.20 AS binary ARG ALPINE_REPO_URL @@ -17,7 +17,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION -FROM alpine:3.18 +FROM alpine:3.20 COPY --from=binary /go/bin/dockerize /usr/local/bin ARG ALPINE_REPO_URL diff --git a/docker/profiles/docker-compose.gms.yml b/docker/profiles/docker-compose.gms.yml index 76bdcacd2dfc9..8cfff2280e2fe 100644 --- a/docker/profiles/docker-compose.gms.yml +++ b/docker/profiles/docker-compose.gms.yml @@ -99,8 +99,7 @@ x-datahub-gms-service: &datahub-gms-service - ${DATAHUB_LOCAL_GMS_ENV:-empty2.env} environment: &datahub-gms-env <<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *datahub-quickstart-telemetry-env, *kafka-env] - ELASTICSEARCH_QUERY_CUSTOM_CONFIG_ENABLED: true - ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE: '/etc/datahub/search/search_config.yaml' + ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE: ${ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE:-search_config.yaml} healthcheck: test: curl -sS --fail http://datahub-gms:${DATAHUB_GMS_PORT:-8080}/health start_period: 90s @@ -119,8 +118,13 @@ x-datahub-gms-service-dev: &datahub-gms-service-dev ports: - ${DATAHUB_MAPPED_GMS_DEBUG_PORT:-5001}:5001 - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 + env_file: + - datahub-gms/env/docker.env + - ${DATAHUB_LOCAL_COMMON_ENV:-empty.env} + - ${DATAHUB_LOCAL_GMS_ENV:-empty2.env} environment: &datahub-gms-dev-env <<: [*datahub-dev-telemetry-env, *datahub-gms-env] + ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE: ${ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE:-/etc/datahub/search/search_config.yaml} SKIP_ELASTICSEARCH_CHECK: false JAVA_TOOL_OPTIONS: '-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5001' BOOTSTRAP_SYSTEM_UPDATE_WAIT_FOR_SYSTEM_UPDATE: false diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml index 25779d7f058ea..834d55096468f 100644 --- a/docker/quickstart/docker-compose-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-m1.quickstart.yml @@ -24,7 +24,7 @@ services: test: nc -z broker $${DATAHUB_KAFKA_BROKER_PORT:-9092} timeout: 5s hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: @@ -271,7 +271,7 @@ services: test: nc -z schema-registry ${DATAHUB_SCHEMA_REGISTRY_PORT:-8081} timeout: 5s hostname: schema-registry - image: confluentinc/cp-schema-registry:7.4.0 + image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: @@ -285,7 +285,7 @@ services: test: echo srvr | nc zookeeper $${DATAHUB_ZK_PORT:-2181} timeout: 5s hostname: zookeeper - image: confluentinc/cp-zookeeper:7.4.0 + image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index 8e3b4b166ce9c..47fb50f78e4f0 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -24,7 +24,7 @@ services: test: nc -z broker $${DATAHUB_KAFKA_BROKER_PORT:-9092} timeout: 5s hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: @@ -245,7 +245,7 @@ services: test: nc -z schema-registry ${DATAHUB_SCHEMA_REGISTRY_PORT:-8081} timeout: 5s hostname: schema-registry - image: confluentinc/cp-schema-registry:7.4.0 + image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: @@ -259,7 +259,7 @@ services: test: echo srvr | nc zookeeper $${DATAHUB_ZK_PORT:-2181} timeout: 5s hostname: zookeeper - image: confluentinc/cp-zookeeper:7.4.0 + image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index d2c2be7e8948d..3fa13a9e56b42 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -24,7 +24,7 @@ services: test: nc -z broker $${DATAHUB_KAFKA_BROKER_PORT:-9092} timeout: 5s hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: @@ -245,7 +245,7 @@ services: test: nc -z schema-registry ${DATAHUB_SCHEMA_REGISTRY_PORT:-8081} timeout: 5s hostname: schema-registry - image: confluentinc/cp-schema-registry:7.4.0 + image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: @@ -259,7 +259,7 @@ services: test: echo srvr | nc zookeeper $${DATAHUB_ZK_PORT:-2181} timeout: 5s hostname: zookeeper - image: confluentinc/cp-zookeeper:7.4.0 + image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index dfe3c5f4f905a..c63b6d1d61b03 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -24,7 +24,7 @@ services: test: nc -z broker $${DATAHUB_KAFKA_BROKER_PORT:-9092} timeout: 5s hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: @@ -271,7 +271,7 @@ services: test: nc -z schema-registry ${DATAHUB_SCHEMA_REGISTRY_PORT:-8081} timeout: 5s hostname: schema-registry - image: confluentinc/cp-schema-registry:7.4.0 + image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: @@ -285,7 +285,7 @@ services: test: echo srvr | nc zookeeper $${DATAHUB_ZK_PORT:-2181} timeout: 5s hostname: zookeeper - image: confluentinc/cp-zookeeper:7.4.0 + image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: diff --git a/docs-website/build.gradle b/docs-website/build.gradle index f3bedd2516319..798047a562ffd 100644 --- a/docs-website/build.gradle +++ b/docs-website/build.gradle @@ -148,8 +148,12 @@ clean { delete 'tmp' delete 'build' delete 'just' + delete 'sphinx/venv' + delete 'sphinx/_build' + delete 'versioned_docs' delete fileTree(dir: 'genDocs', exclude: '.gitignore') delete fileTree(dir: 'docs', exclude: '.gitignore') + delete fileTree(dir: 'genStatic', exclude: '.gitignore') delete 'graphql/combined.graphql' yarnClear } diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js index 892f2b2d14576..df23d1bbf38ba 100644 --- a/docs-website/docusaurus.config.js +++ b/docs-website/docusaurus.config.js @@ -19,6 +19,11 @@ module.exports = { async: true, defer: true, }, + { + src: "/scripts/rb2b.js", + async: true, + defer: true, + } ], noIndex: isSaas, customFields: { @@ -71,6 +76,12 @@ module.exports = { label: "Docs", position: "right", }, + { + to: "/learn", + activeBasePath: "learn", + label: "Learn", + position: "right", + }, { to: "/integrations", activeBasePath: "integrations", @@ -98,6 +109,10 @@ module.exports = { to: "/champions", label: "Champions", }, + { + label: "Share Your Journey", + href: "/customer-stories-survey", + }, ], }, { @@ -139,48 +154,48 @@ module.exports = { type: "docsVersionDropdown", position: "left", dropdownActiveClassDisabled: true, - dropdownItemsAfter: [ - { - type: 'html', - value: '
', - }, - { - type: 'html', - value: '
Archived versions
', - }, - { - value: ` + dropdownItemsAfter: [ + { + type: "html", + value: '
', + }, + { + type: "html", + value: '
Archived versions
', + }, + { + value: ` 0.13.0 `, - type: "html", - }, - { - value: ` + type: "html", + }, + { + value: ` 0.12.1 `, - type: "html", - }, - { - value: ` + type: "html", + }, + { + value: ` 0.11.0 `, - type: "html", - }, - { - value: ` + type: "html", + }, + { + value: ` 0.10.5 `, - type: "html", - }, - ], + type: "html", + }, + ], }, ], }, @@ -273,13 +288,13 @@ module.exports = { "@docusaurus/preset-classic", { docs: { - lastVersion: "current", - versions: { - current: { - label: "Next", - banner: 'none', - }, - }, + lastVersion: "current", + versions: { + current: { + label: "Next", + banner: "none", + }, + }, path: "genDocs", sidebarPath: require.resolve("./sidebars.js"), ...(!isSaas && { @@ -290,7 +305,15 @@ module.exports = { showLastUpdateAuthor: false, showLastUpdateTime: false, }, - blog: false, + blog: { + blogTitle: "DataHub Learn", + blogSidebarTitle: "DataHub Learn", + blogDescription: "Learn about the hot topics in the data ecosystem and how DataHub can help you with your data journey.", + path: "src/learn", + routeBasePath: "learn", + postsPerPage: "ALL", + blogListComponent: "../src/learn/_components/LearnListPage", + }, theme: { customCss: [ isSaas ? require.resolve("./src/styles/acryl.scss") : require.resolve("./src/styles/datahub.scss"), @@ -304,23 +327,23 @@ module.exports = { mdxPageComponent: "@theme/MDXPage", }, googleTagManager: { - containerId: 'GTM-WK28RLTG', + containerId: "GTM-WK28RLTG", }, }, ], ], plugins: [ [ - '@docusaurus/plugin-client-redirects', + "@docusaurus/plugin-client-redirects", { createRedirects(existingPath) { - if (existingPath.includes('/docs')) { + if (existingPath.includes("/docs")) { return [ - existingPath.replace('/docs', '/docs/next'), - existingPath.replace('/docs', '/docs/0.13.0'), - existingPath.replace('/docs', '/docs/0.12.1'), - existingPath.replace('/docs', '/docs/0.11.0'), - existingPath.replace('/docs', '/docs/0.10.5'), + existingPath.replace("/docs", "/docs/next"), + existingPath.replace("/docs", "/docs/0.13.0"), + existingPath.replace("/docs", "/docs/0.12.1"), + existingPath.replace("/docs", "/docs/0.11.0"), + existingPath.replace("/docs", "/docs/0.10.5"), ]; } return undefined; // Return a falsy value: no redirect created diff --git a/docs-website/filterTagIndexes.json b/docs-website/filterTagIndexes.json index 0c1f541cf53d3..8caff3497a200 100644 --- a/docs-website/filterTagIndexes.json +++ b/docs-website/filterTagIndexes.json @@ -67,7 +67,7 @@ } }, { - "Path": "docs/generated/ingestion/sources/csv", + "Path": "docs/generated/ingestion/sources/csv-enricher", "imgPath": "img/datahub-logo-color-mark.svg", "Title": "CSV", "Description": "An ingestion source for enriching metadata provided in CSV format provided by DataHub", @@ -177,7 +177,7 @@ } }, { - "Path": "docs/generated/ingestion/sources/file", + "Path": "docs/generated/ingestion/sources/metadata-file", "imgPath": "img/datahub-logo-color-mark.svg", "Title": "File", "Description": "An ingestion source for single files provided by DataHub", diff --git a/docs-website/graphql/generateGraphQLSchema.sh b/docs-website/graphql/generateGraphQLSchema.sh index da14fbc337f90..a904a2e36d7c1 100755 --- a/docs-website/graphql/generateGraphQLSchema.sh +++ b/docs-website/graphql/generateGraphQLSchema.sh @@ -9,6 +9,7 @@ cat ../../datahub-graphql-core/src/main/resources/app.graphql >> combined.graphq cat ../../datahub-graphql-core/src/main/resources/auth.graphql >> combined.graphql cat ../../datahub-graphql-core/src/main/resources/constraints.graphql >> combined.graphql cat ../../datahub-graphql-core/src/main/resources/entity.graphql >> combined.graphql +cat ../../datahub-graphql-core/src/main/resources/assertions.graphql >> combined.graphql cat ../../datahub-graphql-core/src/main/resources/ingestion.graphql >> combined.graphql cat ../../datahub-graphql-core/src/main/resources/recommendation.graphql >> combined.graphql cat ../../datahub-graphql-core/src/main/resources/search.graphql >> combined.graphql diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 1f2208bc17978..b3dc62a18ac9e 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -31,6 +31,7 @@ module.exports = { label: "Demo", href: "https://demo.datahubproject.io/", }, + "docs/what-is-datahub/customer-stories", "docs/what-is-datahub/datahub-concepts", ], }, @@ -79,6 +80,18 @@ module.exports = { id: "docs/managed-datahub/observe/volume-assertions", className: "saasOnly", }, + { + label: "Open Assertions Specification", + type: "category", + link: { type: "doc", id: "docs/assertions/open-assertions-spec" }, + items: [ + { + label: "Snowflake", + type: "doc", + id: "docs/assertions/snowflake/snowflake_dmfs", + }, + ], + }, ], }, { @@ -206,11 +219,6 @@ module.exports = { id: "docs/managed-datahub/approval-workflows", className: "saasOnly", }, - { - "Metadata Ingestion With Acryl": [ - "docs/managed-datahub/metadata-ingestion-with-acryl/ingestion", - ], - }, { "DataHub API": [ { @@ -264,6 +272,7 @@ module.exports = { }, { "Managed DataHub Release History": [ + "docs/managed-datahub/release-notes/v_0_3_3", "docs/managed-datahub/release-notes/v_0_3_2", "docs/managed-datahub/release-notes/v_0_3_1", "docs/managed-datahub/release-notes/v_0_2_16", @@ -754,7 +763,7 @@ module.exports = { }, { type: "category", - label: "Datahub Actions", + label: "DataHub Actions", link: { type: "doc", id: "docs/act-on-metadata" }, items: [ "docs/actions/README", @@ -800,6 +809,12 @@ module.exports = { "docs/api/tutorials/datasets", "docs/api/tutorials/deprecation", "docs/api/tutorials/descriptions", + "docs/api/tutorials/custom-properties", + "docs/api/tutorials/assertions", + "docs/api/tutorials/custom-assertions", + "docs/api/tutorials/incidents", + "docs/api/tutorials/operations", + "docs/api/tutorials/data-contracts", "docs/api/tutorials/domains", "docs/api/tutorials/forms", "docs/api/tutorials/lineage", @@ -895,6 +910,11 @@ module.exports = { // "docs/_api-guide-template" // - "metadata-service/services/README" // "metadata-ingestion/examples/structured_properties/README" + // "docs/use_cases/business-glossary", + // "docs/use_cases/business-metric", + // "docs/use_cases/data-freshness", + // "docs/use_cases/data-mesh", + // "docs/use_cases/data-pipeline", // ], ], }; diff --git a/docs-website/src/learn/_components/LearnItemCard/index.jsx b/docs-website/src/learn/_components/LearnItemCard/index.jsx new file mode 100644 index 0000000000000..22ebd6c5017e4 --- /dev/null +++ b/docs-website/src/learn/_components/LearnItemCard/index.jsx @@ -0,0 +1,30 @@ +import React from "react"; +import clsx from "clsx"; +import Link from "@docusaurus/Link"; +import { useBlogPost } from "@docusaurus/theme-common/internal"; +import styles from "./styles.module.scss"; + +export default function LearnItemCard() { + const { metadata } = useBlogPost(); + const { permalink, title, description, formattedDate, frontMatter } = metadata; + return ( +
+ + {frontMatter?.image ? ( +
+ {title} +
+ ) : ( +
+

{title}

+
+ )} +
+
{description}
+
+ +
Published on {formattedDate}
+ +
+ ); +} diff --git a/docs-website/src/learn/_components/LearnItemCard/styles.module.scss b/docs-website/src/learn/_components/LearnItemCard/styles.module.scss new file mode 100644 index 0000000000000..35a5c93c348c2 --- /dev/null +++ b/docs-website/src/learn/_components/LearnItemCard/styles.module.scss @@ -0,0 +1,53 @@ +.featureCol { + display: flex; +} + +.card_date { + padding: 1rem 2rem; + font-size: 0.8rem; + font-style: italic; + color: gray; + margin-top: auto; +} + +.card_feature { + font-size: 2rem; + font-weight: 700; +} + +.card { + color: var(--ifm-text-color); + text-decoration: none !important; + padding: 0rem; + margin-bottom: 2rem; + align-self: stretch; + flex-grow: 1; + &:hover { + border-color: var(--ifm-color-primary); + } + hr { + margin: 0; + } +} + +.featureHeader { + h2 { + margin-bottom: 1rem !important; + font-size: 1.25rem; + } + padding: 1rem 2rem; +} + +.featureBody { + padding: 0 2rem; +} + +.card_image { + margin: 0; + margin-bottom: 0.5rem; + + img { + width: 100%; + height: auto; + } +} diff --git a/docs-website/src/learn/_components/LearnListPage/index.jsx b/docs-website/src/learn/_components/LearnListPage/index.jsx new file mode 100644 index 0000000000000..7ce812288df75 --- /dev/null +++ b/docs-website/src/learn/_components/LearnListPage/index.jsx @@ -0,0 +1,91 @@ +import React, { useState } from "react"; +import clsx from "clsx"; + +import useDocusaurusContext from "@docusaurus/useDocusaurusContext"; +import { PageMetadata, HtmlClassNameProvider, ThemeClassNames } from "@docusaurus/theme-common"; +import BlogListPaginator from "@theme/BlogListPaginator"; +import SearchMetadata from "@theme/SearchMetadata"; +import { BlogPostProvider } from "@docusaurus/theme-common/internal"; +import LearnItemCard from "../LearnItemCard"; +import Layout from "@theme/Layout"; +import styles from "./styles.module.scss"; + +function BlogListPageMetadata(props) { + const { metadata } = props; + const { + siteConfig: { title: siteTitle }, + } = useDocusaurusContext(); + const { blogDescription, blogTitle, permalink } = metadata; + const isBlogOnlyMode = permalink === "/"; + const title = isBlogOnlyMode ? siteTitle : blogTitle; + return ( + <> + + + + ); +} + +function BlogListPageContent(props) { + const { metadata, items } = props; + const [activeFilters, setActiveFilters] = useState([]); + // These are currently hardcoded, check the frontmatter of the blog posts to see what audiences are available + const audiences = ["Data Governance Leads", "Data Engineers", "Data Architects", "Data Platform Leads", "Data Analysts"]; + + const filteredItems = activeFilters?.length + ? (items || []).filter((post) => activeFilters.some((activeFilter) => post?.content?.frontMatter?.audience?.some((a) => a === activeFilter))) + : items; + + const handleFilterToggle = (audience) => { + if (activeFilters.includes(audience)) { + setActiveFilters(activeFilters.filter((filter) => filter !== audience)); + } else { + setActiveFilters([...new Set([...activeFilters, audience])]); + } + }; + + return ( + +
+
+
+
+

DataHub Learn

+

Learn about the hot topics in the data ecosystem and how DataHub can help you with your data journey.

+
+
+
+ For: + {audiences.map((audience) => ( + + ))} +
+
+
+
+
+ {(filteredItems || []).map(({ content: BlogPostContent }) => ( + + + + ))} +
+ +
+ + ); +} + +export default function BlogListPage(props) { + return ( + + + + + ); +} diff --git a/docs-website/src/learn/_components/LearnListPage/styles.module.scss b/docs-website/src/learn/_components/LearnListPage/styles.module.scss new file mode 100644 index 0000000000000..514811b0cfa9a --- /dev/null +++ b/docs-website/src/learn/_components/LearnListPage/styles.module.scss @@ -0,0 +1,7 @@ +.filterBar { + display: flex; + justify-content: center; + align-items: center; + gap: 10px; + flex-wrap: wrap; +} diff --git a/docs-website/src/learn/business-glossary.md b/docs-website/src/learn/business-glossary.md new file mode 100644 index 0000000000000..31254f7f1d538 --- /dev/null +++ b/docs-website/src/learn/business-glossary.md @@ -0,0 +1,120 @@ +--- +title: "What is a Business Glossary and How to Standardize It" +description: Understand how a standardized business glossary aids in achieving consistency, compliance, and efficient data use. +tags: ["Business Glossary", "Use Case", "For Data Governance Leads"] +image: /img/learn/use-case-business-glossary.png +hide_table_of_contents: false +audience: ["Data Governance Leads"] +date: 2024-06-03T05:00 +--- + +# What is a Business Glossary and How to Standardize It + +Understand how a standardized business glossary aids in achieving consistency, compliance, and efficient data use. + + + +## Introduction + +Have you ever faced confusion due to inconsistent business terminology within your organization? This lack of standardization can lead to misunderstandings, compliance issues, and inefficient data use. In this post, we’ll explore the importance of having a standardized business glossary, its benefits, and how you can implement one effectively in your organization. + +## What is a Business Glossary? + +A Business Glossary is like a dictionary for your company. It contains definitions of key business terms that everyone in the organization uses, ensuring everyone speaks the same language, especially when it comes to important concepts related to the data your company collects, processes, and uses. + +For example, below are some sales-related glossary terms that can be used in an IT company. + +| Term | Definition | Usage | +| --- | --- | --- | +| CRM (Customer Relationship Management) | Software that manages a company's interactions with current and potential customers. | CRMs help streamline processes and improve customer relationships. | +| Lead | A potential customer who has shown interest in a company's product or service. | Leads are nurtured by the sales team to convert into customers. | +| Pipeline | The stages through which a sales prospect moves from initial contact to final sale. | Sales pipelines track progress and forecast future sales. | +| Quota | A sales target set for a salesperson or team for a specific period. | Quotas motivate sales teams and measure performance. | +| Conversion Rate | The percentage of leads that turn into actual sales. | High conversion rates indicate effective sales strategies. | +| Upselling | Encouraging customers to purchase a more expensive or upgraded version of a product. | Upselling increases revenue by enhancing the customer purchase. | +| Churn Rate | The percentage of customers who stop using a product or service over a given period. | Reducing churn rate is crucial for maintaining steady growth. | +| MQL (Marketing Qualified Lead) | A lead that has been deemed more likely to become a customer based on marketing efforts. | MQLs are passed from the marketing team to the sales team for further nurturing. | +| ARR (Annual Recurring Revenue) | The amount of revenue that a company expects to receive from its customers on an annual basis for subscriptions. | ARR helps in financial forecasting and performance measurement. | + +## What is Business Glossary Standardization? + +Business glossary standardization means creating and maintaining a consistent set of business terms and definitions used across the organization. This practice is essential for maintaining clarity and consistency in how data is interpreted and used across different departments. + +## Why Should You Care? + +### The Challenge + +Without a consistent understanding and use of business terminology, your company lacks a unified understanding of its data. This can lead to inconsistencies, increased compliance risk, and less effective use of data. Different teams may describe the same concepts in various ways, causing confusion about customers, key metrics, products, marketing, and more. + +### The Benefits + +For a governance lead, standardizing the business glossary is crucial for several reasons: + +- **Reduces Confusion, Facilitates Discovery:** Ensures data quality, consistency, and reliability, which are critical for effective decision-making. +- **Regulatory Compliance:** Aligns data use with regulatory definitions and requirements, essential for compliance with financial regulations. +- **Supports Risk Management:** Provides consistent terminology for analyzing market trends, credit risk, and operational risks. +- **Training and Onboarding:** Helps new employees quickly understand the company’s specific language and metrics, speeding up the training process. + +### Real-World Impact + +Imagine a financial services company where different teams use varied terminologies for the same concepts, such as "customer lifetime value." (CLV) This inconsistency can lead to misinterpretations, faulty risk assessments, and regulatory non-compliance, ultimately affecting the company's reputation and financial stability. + +Here's how different teams might interpret CLV and the potential implications: + +| Team | Interpretation of CLV | Focus | Implications | +| --- | --- | --- | --- | +| Marketing | Total revenue generated from a customer over their entire relationship with the company | Campaign effectiveness, customer acquisition costs, return on marketing investment | Revenue maximization through frequent promotions, potentially ignoring the cost of service and risk associated with certain customer segments | +| Sales | Projected future sales from a customer based on past purchasing behavior | Sales targets, customer retention, cross-selling/up-selling opportunities | Aggressive sales tactics to boost short-term sales, potentially leading to customer churn if the value delivered does not meet | +| Finance | Net present value (NPV), factoring in the time value of money and associated costs over the customer relationship period | Profitability, cost management, financial forecasting | Conservative growth strategies, focusing on high-value, low-risk customers, potentially overlooking opportunities for broader market expansion | + + Different interpretations can lead to conflicting strategies and objectives across teams. For instance, Marketing’s aggressive acquisition strategy may lead to a significant increase in new customers and short-term revenue. However, if Finance’s NPV analysis reveals that these customers are not profitable long-term, the company may face financial strain due to high acquisition costs and low profitability. + + The Sales team’s push for upselling may generate short-term sales increases, aligning with their CLV projections. However, if customers feel pressured and perceive the upsells as unnecessary, this could lead to dissatisfaction and higher churn rates, ultimately reducing the actual lifetime value of these customers. + + The conflicting strategies can result in misaligned priorities, where Marketing focuses on volume, Sales on immediate revenue, and Finance on long-term profitability. This misalignment can lead to inefficient resource allocation, where Marketing spends heavily on acquisition, Sales focuses on short-term gains, and Finance restricts budgets due to profitability concerns. + +### Example Discovery Questions + +- Have you ever experienced confusion or errors due to inconsistent terminology in your organization's data reports? How do you currently manage and standardize business terms across departments? +- If your organization lacks a standardized business glossary, what challenges do you face in ensuring regulatory compliance and reliable data analysis? +- When onboarding new employees, do you find that inconsistent terminology slows down their training and understanding of company data? How could a standardized glossary improve this process? + +## How to Standardize a Business Glossary + +### General Approach + +To standardize a business glossary, start by identifying key business terms and their definitions. Engage stakeholders from various departments to ensure comprehensive coverage and agreement. Regularly update the glossary to reflect changes in business processes and regulatory requirements. + +### Alternatives and Best Practices + +Some companies use manual methods to track data terminology and manage access requests. While these methods can work, they are often inefficient and error-prone. Best practices include using automated tools that provide consistent updates and easy access to the glossary for all employees. + +### Our Solution + +Acryl DataHub offers comprehensive features designed to support the authoring of a unified business glossary for your organization: + +

+ +
+ Business Glossary Center +

+ +- **[Centralized Business Glossary](https://datahubproject.io/docs/glossary/business-glossary):** A repository for all business terms and definitions, ensuring consistency across the organization. + + +

+ +
+ Approval Flows +

+ + +- **[Approval Flows](https://datahubproject.io/docs/managed-datahub/approval-workflows):** Structured workflows for approving changes to the glossary, maintaining quality and consistency through time + +- **Automated Data Classification:** Tools to tag critical data assets - tables, columns, dashboards, and pipelines - with terms from the business glossary using automations and custom rules. + +By implementing these solutions, you can ensure that your business terminology is consistently defined and accurately used across all teams, supporting reliable decision-making and regulatory compliance. + +## Conclusion + +Standardizing your business glossary is essential for maintaining consistency, ensuring compliance, and optimizing data use. By implementing best practices and leveraging advanced tools, you can achieve a more efficient and reliable data management process. This investment will lead to better decision-making, reduced compliance risks, and a more cohesive organizational understanding of data. \ No newline at end of file diff --git a/docs-website/src/learn/business-metric.md b/docs-website/src/learn/business-metric.md new file mode 100644 index 0000000000000..523785ced220f --- /dev/null +++ b/docs-website/src/learn/business-metric.md @@ -0,0 +1,83 @@ +--- +title: "What is a Business Metric and How to Define and Standardize Them" +description: Learn the importance of consistent metric definitions and calculation methods to ensure organizational alignment. +tags: ["Business Metric", "Use Case", "For Data Analysts"] +image: /img/learn/use-case-business-metric.png +hide_table_of_contents: false +audience: ["Data Analysts"] +date: 2024-06-03T04:00 +--- + +# What is a Business Metric and How to Define and Standardize Them + +Learn the importance of consistent metric definitions and calculation methods to ensure organizational alignment. + + + +## Introduction + +Have you ever been part of a project where different teams had conflicting definitions for key business metrics like revenue or churn? This misalignment can cause significant issues, leading to incorrect analysis and poor decision-making. In this post, we will explore the importance of defining and standardizing business metrics, why it matters, and how you can achieve it effectively within your organization. + +## What is Business Metrics Definition and Standardization? + +Business metrics definition and standardization involve creating consistent and universally understood definitions for key performance indicators (KPIs) across an organization. Think of it as creating a common language that everyone in your company can use when discussing critical metrics like revenue, churn, or engagement. This ensures that all teams are on the same page, which is essential for accurate analysis and strategic decision-making. + +## Why Should You Care About Business Metrics Definition and Standardization? + +### The Problem + +Data analysts often collaborate with product and finance teams to clarify metrics' conception, calculation methods, and data sources. When there is no consistent understanding of these metrics, it can lead to significant discrepancies in data interpretation and decision-making. Analysts need to ensure that all business-facing metrics use consistent foundational metrics. For example, while a foundational metrics team owns core metrics like revenue and churn, business-facing teams might own engagement metrics. However, they must use the correct definitions for core metrics to analyze the impact of engagement on revenue accurately. + +### Importance + +For data analysts, standardizing business metrics is crucial because these metrics are direct indicators of the performance and health of various functions within an organization. Standardized metrics provide immediate insight into whether the business is on track to meet its objectives. This standardization is vital for top-level company metrics and often necessary for secondary metrics as well. + +### Real-World Impact + +Consider a scenario where the finance team defines revenue differently from the product team. If these discrepancies are not reconciled, it could lead to conflicting reports and misguided strategies. For instance, a marketing campaign analyzed with inconsistent metrics might appear successful in one report and unsuccessful in another, causing confusion and potentially leading to incorrect strategic decisions. + +### Example Discovery Questions and Explanations + +- **Current Management and Challenges:** "How do you currently manage and standardize definitions for core business metrics across different teams, and what challenges have you encountered in this process?" This question helps to uncover the existing processes and pain points in managing metrics, providing insights into potential areas where our product can offer significant improvements. +- **Impact of Misalignment:** "Can you describe a recent instance where misalignment on metric definitions impacted a business decision or analysis, and how was the issue resolved?" This question aims to highlight the real-world consequences of not having standardized metrics, emphasizing the importance of our solution in preventing such issues. +- **Centralized System Importance:** "How important is it for your organization to have a centralized system for defining, accessing, and approving changes to business metrics, and how would you envision such a system supporting your role?" This question seeks to understand the perceived value of a centralized system from the user's perspective, allowing us to tailor our product features to meet their needs effectively. + +## How to Define and Standardize Business Metrics + +### General Approach + +Start by identifying key business metrics and involve stakeholders from different departments to agree on standard definitions. Document these definitions and ensure they are easily accessible to everyone in the organization. Regular reviews and updates are necessary to keep the metrics relevant and aligned with business goals. + +### Alternatives and Best Practices + +Some companies try to align metric definitions through emails and meetings. While this can work, it is often inefficient and prone to misunderstandings. Instead, best practices include using a centralized system for defining and accessing business metrics. Implementing approval flows and lineage tracking can ensure that all changes are reviewed and that the origins of each metric are clear. + +### Our Solution + +Our product offers comprehensive features designed to tackle the challenges of defining and standardizing business metrics: + +

+ +

+ + +- **[Business Glossary](https://datahubproject.io/docs/glossary/business-glossary):** A centralized repository for all metrics definitions, ensuring consistency across the organization. + +

+ +

+ +- **[Approval Flows](https://datahubproject.io/docs/managed-datahub/approval-workflows):** Structured workflows for approving changes to metric definitions, maintaining accuracy and reliability. + +

+ +

+ +- **[Lineage Tracking](https://datahubproject.io/docs/generated/lineage/lineage-feature-guide):** Tools to track the origin and transformations of metrics, ensuring they align with standardized definitions. + + +By implementing these solutions, you can ensure that your business metrics are consistently defined and accurately used across all teams, supporting reliable analysis and decision-making. + +## Conclusion + +Defining and standardizing business metrics is essential for ensuring consistent, accurate, and reliable data analysis and decision-making within an organization. By implementing best practices and leveraging advanced tools like our product’s business glossary, approval flows, and lineage tracking, you can achieve a more cohesive and efficient approach to managing business metrics. This investment will lead to better insights, more informed decisions, and ultimately, a more successful organization. \ No newline at end of file diff --git a/docs-website/src/learn/data-freshness.md b/docs-website/src/learn/data-freshness.md new file mode 100644 index 0000000000000..e635e508fff22 --- /dev/null +++ b/docs-website/src/learn/data-freshness.md @@ -0,0 +1,121 @@ +--- +title: "Ensuring Data Freshness: Why It Matters and How to Achieve It" +description: Explore the significance of maintaining up-to-date data, the challenges involved, and how our solutions can ensure your data remains fresh to meet SLAs. +tags: ["Data Freshness", "Use Case", "For Data Engineers"] +image: /img/learn/use-case-data-freshness.png +hide_table_of_contents: false +audience: ["Data Engineers"] +date: 2024-06-03T01:00 +--- + +# Ensuring Data Freshness: Why It Matters and How to Achieve It + +Explore the significance of maintaining up-to-date data, the challenges involved, and how our solutions can ensure your data remains fresh to meet SLAs. + + + +## Introduction + +Have you ever experienced delays in delivering tables that or machine learning (ML) models that directly power customer experiences due to stale data? Ensuring timely data is crucial for maintaining the effectiveness and reliability of these mission-critical products. In this post, we'll explore the importance of data freshness, the challenges associated with it, and how DataHub can help you meet your data freshness SLAs consistently. + +## What is Data Freshness? + +Data freshness refers to the timeliness and completeness of data used to build tables and ML models. Specifically, freshness can be measured by the difference in time between when some event *actually occurs* vs when that record of that event is reflected in a dataset or used to train an AI model. + +To make things concrete, let’s imagine you run an e-commerce business selling t-shirts. When a user clicks the final “purchase” button to finalize a purchase, this interaction is recorded, eventually winding up in a consolidated “click_events” table on your data warehouse. Data freshness in this case could be measured by comparing when the actual click was performed against when the record of the click landed in the data warehouse. In reality, freshness can be measured against any reference point - e.g. event time, ingestion time, or something else - in relation to when a target table, model, or other data product is updated with new data. + +

+ +
+ Data Freshness +

+ +Oftentimes, data pipelines are designed in order meet some well-defined availability latency, or data freshness SLA, with the specifics of this type of agreement dictating how and when the data pipeline is triggered to run. + +In the modern data landscape, ensuring that data is up-to-date is vital for building high-quality data products, from reporting dashboards used to drive day-to-day company decisions to personalized and dynamic data- or AI-powered product experiences. + +## Why Data Freshness Matters + +For many organizations, fresh data is more than a ‘nice to have’. + +Mission-critical ML models, like those used for price prediction or fraud detection, depend heavily on fresh data to make accurate predictions. Delays in updating these models can lead to lost revenue and damage to your company's reputation. + +Customer-facing data products, for example recommendation features, also need timely updates to ensure that customers receive the most recent and relevant information personalized to them. Delays in data freshness can result in customer frustration, user churn, and loss of trust. + +### Key Considerations for Your Organization + +**Critical Data and ML Models:** + +Can you recall examples when your organization faced challenges in maintaining the timeliness of mission-critical datasets and ML models? If your organization relies on data to deliver concrete product experiences, compliance auditing, or for making high-quality day-to-day decision, then stale data can significantly impact revenue and customer satisfaction. Consider identifying which datasets and models are most critical to your operations and quantifying the business impact of delays. + +**Impact Identification and Response:** + +Because data is highly interconnected, delays in data freshness can lead to cascading problems, particularly of your organization lacks a robust system for identifying and resolving such problems. How does your organization prioritize and manage such incidents? Processes for quickly identifying and resolving root causes are essential for minimizing negative impacts on revenue and reputation. + +**Automated Freshness Monitoring:** + +If data freshness problems often go undetected for long periods of time, there may be opportunities to automate the detection of such problems for core tables and AI models so that your team is first to know when something goes wrong. + +## How to Ensure Data Freshness + +Ensuring data freshness involves several best practices and strategies. Here’s how you can achieve it: + +### Best Practices and Strategies + +**Data Lineage Tracking:** + +Utilize data lineage tracking to establish a bird’s eye view of data flowing through your systems - a picture of the supply chain of data within your organization. This helps in pinpointing hotspots where delays occur and understanding the full impact of such delays to coordinate an effective response. + +**Automation and Monitoring:** + +Implement automated freshness monitoring to detect and address issues promptly. This reduces the need for manual debugging and allows for quicker response times. It can also help you to establish peace-of-mind by targeting your most impactful assets. + +**Incident Management:** + +Establish clear protocols for incident management to prioritize and resolve data freshness issues effectively. This includes setting up notifications and alerts for timely intervention, and a broader communication strategy to involve all stakeholders (even those downstream) in the case of an issue. + +### Alternatives + +While manual investigation and communication using tools like Slack can help triage issues, they often result in time-consuming, inefficient, and informal processes for addressing data quality issues related to freshness, ultimately leading to lower quality outcomes. Automated freshness incident detection and structured incident management via dedicated data monitoring tools can help improve the situation by providing a single place for detecting, communicating, and coordinating to resolve data freshness issues. + +### How DataHub Can Help + +DataHub offers comprehensive features designed to tackle data freshness challenges: + + +**[End-To-End Data Lineage](https://datahubproject.io/docs/generated/lineage/lineage-feature-guide) and [Impact Analysis](https://datahubproject.io/docs/act-on-metadata/impact-analysis):** Easily track the flow of data through your organization to identify, debug, and resolve delays quickly. +

+ +
+ Data Lineage +

+ + +**Freshness Monitoring & Alerting:** Automatically detect and alert when data freshness issues occur, to ensure timely updates by proactively monitoring key datasets for updates. Check out [Assertions](https://datahubproject.io/docs/managed-datahub/observe/assertions) and [Freshness Assertions](https://datahubproject.io/docs/managed-datahub/observe/freshness-assertions), Available in **Acryl Managed DataHub Only.** + +

+ +
+ Freshness Assertions Results +

+ + +

+ +
+ Smart assertions checks for changes on a cadence based on the Table history, by default using the Audit Log. +

+ + +**[Incident Management](https://datahubproject.io/docs/incidents/incidents)** : Centralize data incident management and begin to effectively triage, prioritize, communicate and resolve data freshness issues to all relevant stakeholders. Check out [subscription & notification](https://datahubproject.io/docs/managed-datahub/subscription-and-notification) features as well. + +

+ +

+ + +By implementing these solutions, you can ensure that your key datasets and models are always up-to-date, maintaining their relevancy, accuracy, and reliability for critical use cases within your organization. + +## Conclusion + +Ensuring data freshness is essential for the performance and reliability of critical datasets and AI/ML models. By understanding the importance of data freshness and implementing best practices and automated solutions, you can effectively manage and mitigate delays, thereby protecting your revenue and reputation. DataHub is designed to help you achieve this, providing the tools and features necessary to keep your data fresh and your operations running smoothly. \ No newline at end of file diff --git a/docs-website/src/learn/data-mesh.md b/docs-website/src/learn/data-mesh.md new file mode 100644 index 0000000000000..fdfa4d67de895 --- /dev/null +++ b/docs-website/src/learn/data-mesh.md @@ -0,0 +1,131 @@ +--- +title: "What is a Data Mesh and How to Implement It in Your Organization" +description: Learn how a data mesh aligns data management with domain expertise, enhancing overall organizational agility. +tags: ["Data Mesh", "Use Case", "For Data Architects", "For Data Platform Leads"] +image: /img/learn/use-case-data-mesh.png +hide_table_of_contents: false +audience: ["Data Architects", "Data Platform Leads"] +date: 2024-06-03T02:00 +--- + +# What is Data Mesh and How to Implement It in Your Organization + +Learn how a data mesh aligns data management with domain expertise, enhancing overall organizational agility. + + + +## Introduction + +Have you faced challenges in managing decentralized data across various business units or domains? Implementing a [Data Mesh](https://martinfowler.com/articles/data-mesh-principles.html) can address these issues, aligning data management with domain expertise and enhancing your organization’s overall agility. In this post, we'll explore what a Data Mesh is, why it's beneficial, and how to implement it effectively within your organization. + +## What is Data Mesh? + +Data Mesh is a decentralized data architecture that shifts the responsibility of data management from a central team to individual business units, or "domains." Each domain in turn produces “data products”, or consumable data artifacts, ensuring that data management is closely aligned with domain-specific expertise. This approach promotes agility, scalability, and the ability to generate insights more effectively. + +If you’re familiar with [Service-Oriented Architectures](https://en.wikipedia.org/wiki/Service-oriented_architecture), i.e. micro-services, this might sound familiar. Data Mesh is a somewhat analogous concept, but applied to data! + +

+ +
+ 4 Principles of Data Mesh +

+ + +| Principle | Explanation | +| --- | --- | +| Domain Data Ownership | Organizing data into explicit domains based on the structure of your organization, and then assigning clear accountability to each. This enables you to more easily increase the number of sources of data, variety of use cases, and diversity of access models to the data increases. | +| Data as a product | Domain data should be highly accessible and highly reliable by default. It should be easy to discover, easy to understand, easy to access securely, and high quality. | +| Self-Service | Domain teams should be able to independently create, consume, and manage data products on top of a general-purpose platform that can hide the complexity of building, executing and maintaining secure and interoperable data products. | +| Federated Governance | Consistent standards that are enforced by process and technology around interoperability, compliance, and quality. This makes it easy for data consumers to interact with data products across domains in familiar way and ensures quality is maintained uniformly. | + +

+ +
+ Logical architecture of data mesh approach, Image Credit: Zhamak Dehghani +

+ + + +## Why Implement Data Mesh? + +For data architects and data platform leads, implementing a Data Mesh can resolve various challenges associated with managing decentralized data, particularly as you try to scale up. + +Traditional data lakes or warehouses can become central bottlenecks, impairing access, understanding, accountability, and quality of data - ultimately, its usability. These architectures can struggle to meet the diverse needs of different business units, leading to inefficiencies. + +Data Mesh addresses these issues by formally dividing data into decentralized domains, which are owned by the individual teams who are experts in those domains. This approach allows each business unit or domain to manage its own data, enabling independent creation and consumption of data and increasing the agility, reliability, scalability of an organization’s data practice. + +### Key Considerations for Your Organization + +**Decentralized Data Management:** Have you experienced difficulties or bottlenecks in managing data across various business units? Implementing a Data Mesh can alleviate these issues by allowing each domain to build and share its own data products, enhancing agility and scalability. + +**Overcoming Centralized Bottlenecks:** If your organization relies on a centralized data lake or warehouse, or data platform team, have you encountered limitations in scalability or delays in data access and analysis? Data Mesh can help overcome these bottlenecks by “pushing down” data ownership and management to domain experts. + +**Enhancing Agility and Reliability:** How important is it for your organization to respond quickly to market changes and generate insights reliably? By formally defining the responsibilities around data “products”, a data mesh architecture can provide the flexibility and speed needed to stay competitive. + +## How to Implement Data Mesh + +Implementing Data Mesh doesn’t need to be a headache. Here’s how your organization can move towards a better future: + +### Best Practices and Strategies + +**Define Domains and Data Products** + +Formally define the different business units or domains within your organization and define the data products each domain will own and manage, and then begin to organize the data on your existing warehouse or lake around these domains. This ensures clarity and responsibility for data management. + +**Establish Clear Contracts** + +Create a clear set of expectations around what it means to be a domain or data product owner within your organization. Then, build processes and systems to both reinforce and monitor these expectations. This helps maintain consistency and reliability across the organization. + +**Monitor Data Quality** + +Use metadata validation and data quality assertions to ensure that your expectations are being met. This includes setting standards for both data quality - freshness, volume, column validity - as well compliance with your less technical requirements - ownership, data documentation, and data classification. + +**Move Towards Federated Governance** + +Adopt a federated governance model to balance autonomy and control. While domains manage their data products, a central team can oversee governance standards and ensure compliance with organizational policies via a well-defined review process. + +### Alternatives + +While a centralized data lake or warehouse can simplify data governance by virtue of keeping everything in one place, it can become a bottleneck as your data organization grows. Decentralized Data Mesh can provide a more scalable and agile approach, by distributing day-to-day responsibility for accessing, producing, and validating data while enforcing a centralized set of standards and processes. + +### Our Solution + +Acryl DataHub offers a comprehensive set of features designed to support the implementation of a Data Mesh at your organization: + +- **[Data Domains](https://datahubproject.io/docs/domains)**: Clearly define and manage data products within each business unit. +- **[Data Products](https://datahubproject.io/docs/dataproducts):** Ensure each domain owns and manages its data products, promoting autonomy and agility. +- **[Data Contracts](https://datahubproject.io/docs/managed-datahub/observe/data-contract)**: Establish clear agreements between domains to ensure consistency and reliability. + + +

+ +
+ Data Contracts in Acryl DataHub UI +

+ + + +- **[Assertions](https://datahubproject.io/docs/managed-datahub/observe/assertions)** Monitor data quality using freshness, volume, column validity, schema, and custom SQL checks to get notified first when things go wrong + + +

+ +
+ Assertion Results +

+ + + +- **[Metadata Tests](https://datahubproject.io/docs/tests/metadata-tests)**: Monitor and enforce a central set of standards or policies across all of your data assets, e.g. to ensure data documentation, data ownership, and data classification. + +

+ +
+ Metadata Test Results +

+ +By implementing these solutions, you can effectively manage decentralized data, enhance agility, and generate insights more efficiently. + +## Conclusion + +Implementing a Data Mesh can significantly improve your organization's ability to manage and leverage decentralized data. By understanding the benefits of data mesh and following best practices for implementation, you can overcome the limitations of centralized data systems and enhance your agility, scalability, and ability to generate insights. Acryl DataHub was built from the ground up to help you achieve this, providing the tools and features necessary to implement a large-scale Data Mesh successfully. \ No newline at end of file diff --git a/docs-website/src/learn/data-pipeline.md b/docs-website/src/learn/data-pipeline.md new file mode 100644 index 0000000000000..f2995c299851b --- /dev/null +++ b/docs-website/src/learn/data-pipeline.md @@ -0,0 +1,90 @@ +--- +title: "What is a Data Pipeline and Why Should We Optimize It" +description: Discover the importance of optimizing data pipelines to maintain data freshness and control costs. +tags: ["Data Pipeline", "Use Case", "For Data Engineers"] +image: /img/learn/use-case-data-pipeline.png +hide_table_of_contents: false +audience: ["Data Engineers"] +date: 2024-06-03T03:00 +--- + +# What is a Data Pipeline and Why Should We Optimize It? + +Discover the importance of optimizing data pipelines to maintain data freshness and control costs. + + + +## Introduction + +Have you ever been frustrated by slow and unreliable data pipelines or unexpectedly high cloud bills? In the modern data world, maintaining efficient, reliable, and cost-effective data pipelines is crucial for delivering timely, high-quality data. This post will explore the importance of optimizing data pipelines, why it matters, and how to achieve it effectively. + +## What is a Data Pipeline? + +A data pipeline is a series of processes that move data from one system to another - a key component in the supply chain for data. Think of it like a conveyor belt in a factory, transporting raw materials to different stations where they are processed into the final product. In the context of data, pipelines extract, transform, and load data (ETL) from various sources to destinations like data warehouses, ensuring the data is ready for analysis and use in applications such as machine learning models and business intelligence dashboards. + + +

+ +
+ Data Pipeline Example +

+ +## Why Should You Care About Data Pipeline Optimization? + +### The Problem + +Over time, data pipelines can slow down or become unreliable due to new dependencies, application code bugs, and poorly optimized queries, leading to missed data freshness SLAs and increased cloud costs. For data engineers, this means more time spent on manual debugging and justifying costs to your executives. + +### Importance + +Efficient data pipelines are essential for maintaining the performance of mission-critical tables, dashboards, and ML models powering key use cases for your organization. For example, a price prediction model relies on timely data to provide accurate results, directly impacting revenue. Similarly, outdated customer data can harm a company’s reputation and customer satisfaction. + +### Real-World Impact + +Imagine you’re managing a recommendation engine for an e-commerce site. If your data pipeline is delayed, the recommendations could become outdated, leading to missed sales opportunities - financial costs - and a poor user experience - reputational costs. Alternatively, consider a fraud detection system that relies on real-time data; any delay or downtime could mean the difference between catching fraudulent activity and suffering significant financial loss. + +### Questions To Ask + +- Have you ever noticed a decline in the freshness of crucial data or an uptick in cloud costs for specific pipelines? How do you currently approach diagnosing and optimizing these pipelines? +- If your organization is facing increasing cloud bills due to data pipeline inefficiencies, what strategies or tools do you employ to monitor and optimize costs? How do you balance the trade-off between performance, cost, and meeting business stakeholders' expectations for data delivery? +- Are you taking proactive measures to prevent data pipelines from becoming slower, more fragile, or more expensive over time? Do you have a system in place for regularly reviewing and optimizing key data pipelines to prevent performance or cost degradation? + +## How to Optimize Data Pipelines + +### General Approach + +To optimize your data pipelines, start by identifying bottlenecks and inefficiencies in the pipelines that generate your most mission-critical tables, dashboards, and models. Regularly review and update queries, and monitor pipeline performance by measuring aggregate pipeline run times as well as more granular tracking at the step or query level to catch issues early. Implement automation wherever possible to reduce manual intervention and ensure consistency. + +### Alternatives and Best Practices + +Some companies resort to manual debugging or use communication tools like Slack to triage issues. While these methods can work, they are often time-consuming and prone to errors. Instead, consider leveraging tools that provide lineage tracking, last updated time, and automated monitoring to streamline the optimization process. + +### Our Solution + +Acryl DataHub offers comprehensive features designed to optimize data pipelines: + +

+ +
+ Pipeline Catalog +

+ +- **Pipeline Cataloging:** Quickly browse all of the data pipelines running inside your organization, and track critical human context like pipeline ownership / accountability, purpose / documentation, and compliance labels in one place. + +

+ +
+ Lineage Tracking +

+ +- **[Lineage Tracking](https://datahubproject.io/docs/generated/lineage/lineage-feature-guide) and [Impact Analysis](https://datahubproject.io/docs/act-on-metadata/impact-analysis):** Understand the flow of data through your pipelines to identify and resolve inefficiencies quickly. Easily see which assets are consumed and produced by which pipelines. +- **Freshness Monitoring:** Track the freshness using Freshness Assertions of your data to ensure SLAs are met consistently. +- **Cost Management Tooling:** Monitor and optimize cloud costs associated with your data pipelines to improve cost-efficiency. + +By implementing these solutions, you can ensure that your data pipelines are running efficiently, meeting delivery SLAs, and staying within budget. + + + +## Conclusion + +Optimizing data pipelines is essential for maintaining data reliability, controlling costs, and ultimately ensuring your business continues to run smoothly. By implementing best practices and leveraging advanced tools like our product’s lineage tracking and automated monitoring features, you can achieve efficient and cost-effective data pipelines. Investing time and resources into optimization will ultimately lead to better performance, lower costs, and more satisfied stakeholders. \ No newline at end of file diff --git a/docs-website/src/pages/champions/index.js b/docs-website/src/pages/champions/index.js index 41d1e48ef6772..b544defc47448 100644 --- a/docs-website/src/pages/champions/index.js +++ b/docs-website/src/pages/champions/index.js @@ -161,7 +161,7 @@ const championSections = [ { name: "Tim Bossenmaier", image: "https://raw.githubusercontent.com/datahub-project/static-assets/champ-img/imgs/datahub-champions/tim_bossenmaier.jpeg", - position: "Data & Software Engineer, Bytefabrik.AI", + position: "Data & Software Engineer, Cloudflight", bio: ( <>

@@ -173,7 +173,7 @@ const championSections = [ linkedin: "https://www.linkedin.com/in/tim-bossenmaier/", github: "https://github.com/bossenti", }, - location: "Sigmaringen, Germany" + location: "Innsbruck, Austria" }, { name: "Nikola Kasev", diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/sql/__init__.py b/docs-website/src/pages/customer-stories-survey/customerstoriessurvey.scss similarity index 100% rename from metadata-ingestion/src/datahub/ingestion/source_config/sql/__init__.py rename to docs-website/src/pages/customer-stories-survey/customerstoriessurvey.scss diff --git a/docs-website/src/pages/customer-stories-survey/index.js b/docs-website/src/pages/customer-stories-survey/index.js new file mode 100644 index 0000000000000..63a3ecd77e968 --- /dev/null +++ b/docs-website/src/pages/customer-stories-survey/index.js @@ -0,0 +1,48 @@ +import React, { useEffect } from 'react'; +import Layout from '@theme/Layout'; +import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; + +function CustomerStoriesSurvey() { + const { siteConfig = {} } = useDocusaurusContext(); + + useEffect(() => { + const script = document.createElement('script'); + script.src = "//js.hsforms.net/forms/embed/v2.js"; + script.async = true; + script.type = 'text/javascript'; + document.body.appendChild(script); + + script.onload = () => { + if (window.hbspt) { + window.hbspt.forms.create({ + region: "na1", + portalId: "14552909", + formId: "087ef03d-e47e-4814-b458-b30e3e02b623", + target: '#hubspotForm' // Targeting the div with the specific ID + }); + } + }; + + return () => { + document.body.removeChild(script); + }; + }, []); + + return ( + +

+
+
+

We Want To Hear Your Journey!

+
Share Your DataHub Journey with Our Community.
+
+
+
+
+ + ); +} + +export default CustomerStoriesSurvey; diff --git a/docs-website/src/pages/docs/_components/CustomerCard/customercard.module.scss b/docs-website/src/pages/docs/_components/CustomerCard/customercard.module.scss new file mode 100644 index 0000000000000..349f705d25b10 --- /dev/null +++ b/docs-website/src/pages/docs/_components/CustomerCard/customercard.module.scss @@ -0,0 +1,56 @@ +.card { + color: var(--ifm-hero-text-color); + padding: 0; + margin: 0rem 3rem 2rem 0rem; + text-decoration: none !important; + + .card_button { + padding: 0rem 0rem 0rem 1rem; + text-align: right; + } + + .card_img { + justify-content: center; + display: flex; + height: 250px; + margin: 0; + position: relative; + text-align: center; + } + + .card_body { + padding: 2rem 3rem 2rem 3rem; + + .card_description { + min-height: 20rem; + } + } + + .card_overlay_text { + position: absolute; + text-align: left; + width: 80%; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + color: white; + + .card_customer { + font-size: 3.2rem; + font-weight: 800; + line-height: 1.2; + + } + .card_title { + font-size: 1.2rem; + font-weight: 600; + } + + } + + img { + object-fit: cover; + filter: brightness(50%); + } + +} diff --git a/docs-website/src/pages/docs/_components/CustomerCard/index.jsx b/docs-website/src/pages/docs/_components/CustomerCard/index.jsx new file mode 100644 index 0000000000000..36c83226e1f73 --- /dev/null +++ b/docs-website/src/pages/docs/_components/CustomerCard/index.jsx @@ -0,0 +1,30 @@ +import React from "react"; +import clsx from "clsx"; +import styles from "./customercard.module.scss"; +import Link from "@docusaurus/Link"; + +const CustomerCard = ({ customer, title, imgUrl, description, to,}) => { + return ( +
+
+
+ {customer} +
+
{customer}
+
{title}
+
+
+
+

{description}

+
+ + Discover {customer}'s Story + +
+
+
+
+ ); +}; + +export default CustomerCard; diff --git a/docs-website/src/pages/docs/_components/CustomerCardSection/customercardsection.module.scss b/docs-website/src/pages/docs/_components/CustomerCardSection/customercardsection.module.scss new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/docs-website/src/pages/docs/_components/CustomerCardSection/index.jsx b/docs-website/src/pages/docs/_components/CustomerCardSection/index.jsx new file mode 100644 index 0000000000000..ca34d89df8701 --- /dev/null +++ b/docs-website/src/pages/docs/_components/CustomerCardSection/index.jsx @@ -0,0 +1,96 @@ +import React from "react"; +import CustomerCard from '../CustomerCard' + +const customerCardContent = [ + { + customer: "Netflix", + imgUrl: "/img/assets/netflix.jpg", + title: "How they are contributing to DataHub to make it more extensible", + description: ( + <> + + "DataHub gave us the extensibility features we needed to define new + entity types easily and augment existing ones. + DataHub performed exceptionally well in managing our traffic load and data + volume. It offers a great developer experience, a well-documented + taxonomy, and — very importantly — solid community support." +
+
+ — Ajoy Majumdar, Software Architect at Netflix + +
+ + ), + to: "https://youtu.be/ejcO7hdX0lk?si=8iPjrPeBZq5KNdb-", + }, + { + customer: "Visa", + imgUrl: "/img/assets/travel.jpg", + title: "How to use DataHub to scale your Data Governance", + description: ( + <> + + "We found DataHub to provide excellent coverage for our needs. What we + appreciate most about DataHub is its powerful API platform." +
+
+ — Jean-Pierre Dijcks, Sr. Dir. Product Management at VISA + +
+ + ), + to: "https://youtu.be/B6CplqnIkFw?si=jrrr04cV5rdxO6Ra", + }, + { + customer: "MediaMarkt Saturn", + imgUrl: "/img/assets/business.jpg", + title: "Building Data Access Management within DataHub", + description: ( + <> + Europe’s #1 consumer electronics retailer implemented DataHub for three reasons: +
+
+ 1. DataHub provides an extremely flexible and customizable metadata platform at scale +
+ 2. Open-source means lower cost to implement and removes the headache of license management +
+ 3. Community-driven project which continually evolves with industry trends and best practices + + ), + to: "https://www.acryldata.io/blog/data-contracts-in-datahub-combining-verifiability-with-holistic-data-management?utm_source=datahub&utm_medium=referral&utm_content=blog", + }, + { + customer: "Airtel", + imgUrl: "/img/assets/phonecall.jpg", + title: "DataHub is the Bedrock of Data Mesh at Airtel", + description: ( + <> + Airtel is a leading global telecommunication provider. DataHub is the + bedrock of Data Mesh at Airtel by providing the requisite governance and + metadata management functionality to ensure their Data Products should + are discoverable, addressable, trustworthy, self-describing, and secure. +
+
+ Get a closer look at how the Airtel team has successfully integrated + DataHub to take their data mesh implementation to the next level. + + ), + to: "https://youtu.be/wsCFnElN_Wo?si=i-bNAQAsbHJq5O9-", + }, +]; + +const CustomerCardSection = () => { + return ( +
+
+
+ {customerCardContent.map((props, idx) => ( + + ))} +
+
+
+ ); +}; + +export default CustomerCardSection; diff --git a/docs-website/src/pages/docs/_components/FilterBar/index.jsx b/docs-website/src/pages/docs/_components/FilterBar/index.jsx index 354ff1cfa7d3d..2d85f976518d0 100644 --- a/docs-website/src/pages/docs/_components/FilterBar/index.jsx +++ b/docs-website/src/pages/docs/_components/FilterBar/index.jsx @@ -24,6 +24,7 @@ function FilterBar({ filterOptions, allowExclusivity, setIsExclusive, + searchPlaceholder }) { const [filtersOpen, setFiltersOpen] = useState(false); const [selectedFilters, setSelectedFilters] = useState([]); @@ -65,7 +66,7 @@ function FilterBar({ className={styles.searchQueryInput} placeholder={translate({ id: "theme.SearchPage.inputPlaceholder", - message: "Filter Integrations", + message: searchPlaceholder, description: "The placeholder for search page input", })} aria-label={translate({ diff --git a/docs-website/src/pages/docs/_components/FilterPage/index.jsx b/docs-website/src/pages/docs/_components/FilterPage/index.jsx index 7c754adb4ca81..42dbe93afb61d 100644 --- a/docs-website/src/pages/docs/_components/FilterPage/index.jsx +++ b/docs-website/src/pages/docs/_components/FilterPage/index.jsx @@ -102,6 +102,7 @@ export function FilterPage( filterOptions={filterOptions} allowExclusivity={allowExclusivity} setIsExclusive={setIsExclusive} + searchPlaceholder="Search for ingestion sources" />
diff --git a/docs-website/static/img/assets/business.jpg b/docs-website/static/img/assets/business.jpg new file mode 100644 index 0000000000000..f5a91928ee2ad Binary files /dev/null and b/docs-website/static/img/assets/business.jpg differ diff --git a/docs-website/static/img/assets/netflix.jpg b/docs-website/static/img/assets/netflix.jpg new file mode 100644 index 0000000000000..8b555f5b63187 Binary files /dev/null and b/docs-website/static/img/assets/netflix.jpg differ diff --git a/docs-website/static/img/assets/phonecall.jpg b/docs-website/static/img/assets/phonecall.jpg new file mode 100644 index 0000000000000..87e48f2821382 Binary files /dev/null and b/docs-website/static/img/assets/phonecall.jpg differ diff --git a/docs-website/static/img/assets/travel.jpg b/docs-website/static/img/assets/travel.jpg new file mode 100644 index 0000000000000..de2697f563121 Binary files /dev/null and b/docs-website/static/img/assets/travel.jpg differ diff --git a/docs-website/static/img/learn/use-case-business-glossary.png b/docs-website/static/img/learn/use-case-business-glossary.png new file mode 100644 index 0000000000000..4783f1fa2ab79 Binary files /dev/null and b/docs-website/static/img/learn/use-case-business-glossary.png differ diff --git a/docs-website/static/img/learn/use-case-business-metric.png b/docs-website/static/img/learn/use-case-business-metric.png new file mode 100644 index 0000000000000..5b4f2494f9ce4 Binary files /dev/null and b/docs-website/static/img/learn/use-case-business-metric.png differ diff --git a/docs-website/static/img/learn/use-case-data-freshness.png b/docs-website/static/img/learn/use-case-data-freshness.png new file mode 100644 index 0000000000000..8878bfabd75bf Binary files /dev/null and b/docs-website/static/img/learn/use-case-data-freshness.png differ diff --git a/docs-website/static/img/learn/use-case-data-mesh.png b/docs-website/static/img/learn/use-case-data-mesh.png new file mode 100644 index 0000000000000..ae5ce1a4cf1c0 Binary files /dev/null and b/docs-website/static/img/learn/use-case-data-mesh.png differ diff --git a/docs-website/static/img/learn/use-case-data-pipeline.png b/docs-website/static/img/learn/use-case-data-pipeline.png new file mode 100644 index 0000000000000..3003f295679bb Binary files /dev/null and b/docs-website/static/img/learn/use-case-data-pipeline.png differ diff --git a/docs-website/static/scripts/rb2b.js b/docs-website/static/scripts/rb2b.js new file mode 100644 index 0000000000000..684c981116bd9 --- /dev/null +++ b/docs-website/static/scripts/rb2b.js @@ -0,0 +1 @@ +!function () {var reb2b = window.reb2b = window.reb2b || [];if (reb2b.invoked) return;reb2b.invoked = true;reb2b.methods = ["identify", "collect"];reb2b.factory = function (method) {return function () {var args = Array.prototype.slice.call(arguments);args.unshift(method);reb2b.push(args);return reb2b;};};for (var i = 0; i < reb2b.methods.length; i++) {var key = reb2b.methods[i];reb2b[key] = reb2b.factory(key);}reb2b.load = function (key) {var script = document.createElement("script");script.type = "text/javascript";script.async = true;script.src = "https://s3-us-west-2.amazonaws.com/b2bjsstore/b/" + key + "/reb2b.js.gz";var first = document.getElementsByTagName("script")[0];first.parentNode.insertBefore(script, first);};reb2b.SNIPPET_VERSION = "1.0.1";reb2b.load("8XOE9GHYQ3OM");}(); \ No newline at end of file diff --git a/docs-website/useCaseIndexes.json b/docs-website/useCaseIndexes.json new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/docs-website/vercel-setup.sh b/docs-website/vercel-setup.sh index 01a1ee65b34f0..4bb40eaddf477 100755 --- a/docs-website/vercel-setup.sh +++ b/docs-website/vercel-setup.sh @@ -4,6 +4,10 @@ set -euxo pipefail ./metadata-ingestion/scripts/install_deps.sh +# Set up java version for gradle +yum install java-17-amazon-corretto -y +java --version + # Build python from source. # Amazon Linux 2 has Python 3.8, but it's version of OpenSSL is super old and hence it # doesn't work with the packages we use. As such, we have to build Python from source. @@ -11,8 +15,7 @@ set -euxo pipefail # for reuse. yum groupinstall "Development Tools" -y -yum erase openssl-devel -y -yum install openssl11 openssl11-devel libffi-devel bzip2-devel wget nodejs -y +yum install openssl openssl-devel libffi-devel bzip2-devel wget nodejs -y wget https://www.python.org/ftp/python/3.10.11/Python-3.10.11.tgz tar -xf Python-3.10.11.tgz @@ -29,6 +32,3 @@ rm "$py3" ln "$(which python3.10)" "$py3" python3 --version -# Set up java version for gradle -yum install java-17-amazon-corretto -java --version \ No newline at end of file diff --git a/docs-website/yarn.lock b/docs-website/yarn.lock index 61bbc305b8917..a93b0e74c327d 100644 --- a/docs-website/yarn.lock +++ b/docs-website/yarn.lock @@ -1827,7 +1827,7 @@ "@docusaurus/theme-search-algolia" "2.4.3" "@docusaurus/types" "2.4.3" -"@docusaurus/react-loadable@5.5.2", "react-loadable@npm:@docusaurus/react-loadable@5.5.2": +"@docusaurus/react-loadable@5.5.2": version "5.5.2" resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce" integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== @@ -4174,11 +4174,11 @@ brace-expansion@^1.1.7: concat-map "0.0.1" braces@^3.0.2, braces@~3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" - integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== + version "3.0.3" + resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" + integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== dependencies: - fill-range "^7.0.1" + fill-range "^7.1.1" browserslist@^4.0.0, browserslist@^4.18.1, browserslist@^4.21.10, browserslist@^4.21.4, browserslist@^4.22.2, browserslist@^4.23.0: version "4.23.0" @@ -5657,10 +5657,10 @@ filesize@^8.0.6: resolved "https://registry.yarnpkg.com/filesize/-/filesize-8.0.7.tgz#695e70d80f4e47012c132d57a059e80c6b580bd8" integrity sha512-pjmC+bkIF8XI7fWaH8KxHcZL3DPybs1roSKP4rKDvy20tAWwIObE4+JIseG2byfGKhud5ZnM4YSGKBz7Sh0ndQ== -fill-range@^7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" - integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ== +fill-range@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" + integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== dependencies: to-regex-range "^5.0.1" @@ -9705,6 +9705,14 @@ react-loadable-ssr-addon-v5-slorber@^1.0.1: dependencies: "@babel/runtime" "^7.10.3" +"react-loadable@npm:@docusaurus/react-loadable@5.5.2": + version "5.5.2" + resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce" + integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== + dependencies: + "@types/react" "*" + prop-types "^15.6.2" + react-markdown@^8.0.6: version "8.0.7" resolved "https://registry.yarnpkg.com/react-markdown/-/react-markdown-8.0.7.tgz#c8dbd1b9ba5f1c5e7e5f2a44de465a3caafdf89b" diff --git a/docs/actions/actions/slack.md b/docs/actions/actions/slack.md index 9ff3ec74f51d0..bdea1c479e8aa 100644 --- a/docs/actions/actions/slack.md +++ b/docs/actions/actions/slack.md @@ -190,7 +190,7 @@ Similar to the quickstart scenario, there are no specific software installation If you are using the `datahub-actions` library directly from Python, or the `datahub-actions` cli directly, then you need to first install the `slack` action plugin in your Python virtualenv. ``` -pip install "datahub-actions[slack]" +pip install "acryl-datahub-actions[slack]" ``` Then run the action with a configuration file that you have modified to capture your credentials and configuration. diff --git a/docs/actions/actions/teams.md b/docs/actions/actions/teams.md index c61f472cf36fe..f14cd951f95e8 100644 --- a/docs/actions/actions/teams.md +++ b/docs/actions/actions/teams.md @@ -95,7 +95,7 @@ Similar to the quickstart scenario, there are no specific software installation If you are using the `datahub-actions` library directly from Python, or the `datahub-actions` cli directly, then you need to first install the `teams` action plugin in your Python virtualenv. ``` -pip install "datahub-actions[teams]" +pip install "acryl-datahub-actions[teams]" ``` Then run the action with a configuration file that you have modified to capture your credentials and configuration. diff --git a/docs/advanced/mcp-mcl.md b/docs/advanced/mcp-mcl.md index 235c9a85ec8da..9efb9b794954d 100644 --- a/docs/advanced/mcp-mcl.md +++ b/docs/advanced/mcp-mcl.md @@ -60,10 +60,14 @@ record MetadataChangeProposal { **/ systemMetadata: optional SystemMetadata + /** + * Headers - intended to mimic http headers + */ + headers: optional map[string, string] } ``` -Each proposal comprises of the following: +Each proposal is comprised of the following: 1. entityType @@ -82,12 +86,13 @@ Each proposal comprises of the following: Type of change you are proposing: one of - UPSERT: Insert if not exists, update otherwise - - CREATE: Insert if not exists, fail otherwise + - CREATE: Insert aspect if not exists, fail otherwise + - CREATE_ENTITY: Insert if entity does not exist, fail otherwise - UPDATE: Update if exists, fail otherwise - DELETE: Delete - PATCH: Patch the aspect instead of doing a full replace - Only UPSERT, CREATE, DELETE, PATCH are supported as of now. + Only UPSERT, CREATE, CREATE_ENTITY, DELETE, PATCH are supported as of now. 5. aspectName @@ -110,6 +115,10 @@ Each proposal comprises of the following: Extra metadata about the proposal like run_id or updated timestamp. +8. headers + + Optional headers which are meant to mimic http headers. These are currently used for implementing conditional write logic. + GMS processes the proposal and produces the Metadata Change Log, which looks like this. ```protobuf @@ -156,4 +165,44 @@ entities: keyAspect: datasetKey aspects: - datasetProfile -``` \ No newline at end of file +``` + +## Features + +### Conditional Writes + +Conditional write semantics use extra information contained in the MCP `headers` field to possibly avoid writing new aspects +if the conditions are not met. + +#### If-Version-Match + +Each time an aspect is updated a `version` is incremented to represent the change to the aspect. This `version` is stored and returned +in `SystemMetadata`. + +A writer can provide a header with the expected `version` when initiating the request. If the expected `version` does not +match the actual `version` stored in the database, the write will fail. This prevents overwriting an aspect that has +been modified by another process. + +Note: If the aspect doesn't exist yet, then the `version` is `-1`. A writer can use this `version` to only create +an aspect if it doesn't. Also see _Change Types: [`CREATE`, `CREATE_ENTITY`]_ section below. + +#### If-Modified-Since / If-Unmodified-Since + +A writer may also specify time-based conditions using http header semantics. Similar to version based conditional writes +this method can be used to prevent the write if the target aspect was modified after a reading the aspect. Per the +http specification dates must comply with ISO-8601 standard. + +`If-Unmodified-Since`: +A writer can specify that the aspect must NOT have been modified after a specific time, following [If-Unmodified-Since](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Unmodified-Since) http headers. + +`If-Modified-Since` +A writer can specify that the aspect must have been modified after a specific time, following [If-Modified-Since](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Modified-Since) http headers. + +#### Change Types: [`CREATE`, `CREATE_ENTITY`] + +Another form of conditional writes which considers the existence of an aspect or entity uses the following Change Types. + +`CREATE` - Create the aspect if it doesn't already exist. + +`CREATE_ENTITY` - Create the aspect if no aspects exist for the entity. + diff --git a/docs/advanced/patch.md b/docs/advanced/patch.md index 21c28941c265a..601d055659313 100644 --- a/docs/advanced/patch.md +++ b/docs/advanced/patch.md @@ -62,3 +62,114 @@ Here are a few illustrative examples using the Python Patch builders: + +## How Patch works + +To understand how patching works, it's important to understand a bit about our [models](../what/aspect.md). Entities are comprised of Aspects +which can be reasoned about as JSON representations of the object models. To be able to patch these we utilize [JsonPatch](https://jsonpatch.com/). The components of a JSON Patch are the path, operation, and value. + +### Path + +The JSON path refers to a value within the schema. This can be a single field or can be an entire object reference depending on what the path is. +For our patches we are primarily targeting single fields or even single array elements within a field. To be able to target array elements by id, we go through a translation process +of the schema to transform arrays into maps. This allows a path to reference a particular array element by key rather than by index, for example a specific tag urn being added to a dataset. +This is important to note that for some fields in our schema that are arrays which do not necessarily restrict uniqueness, this puts a uniqueness constraint on the key. +The key for objects stored in arrays is determined manually by examining the schema and a long term goal is to make these keys annotation driven to reduce the amount of code needed to support +additional aspects to be patched. There is a generic patch endpoint, but it requires any array field keys to be specified at request time, putting a lot of burden on the API user. + +#### Examples + +A patch path for targeting an upstream dataset: + +`/upstreams/urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created_upstream,PROD)` + +Breakdown: +* `/upstreams` -> References the upstreams field of the UpstreamLineage aspect, this is an array of Upstream objects where the key is the Urn +* `/urn:...` -> The dataset to be targeted by the operation + + +A patch path for targeting a fine grained lineage upstream: + +`/fineGrainedLineages/TRANSFORM/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),foo)/urn:li:query:queryId/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created_upstream,PROD),bar)` + +Breakdown: +* `/fineGrainedLineages` -> References the fineGrainedLineages field on an UpstreamLineage, this is an array of FineGrainedLineage objects keyed on transformOperation, downstream urn, and query urn +* `/TRANSFORM` -> transformOperation, one of the fields determining the key for a fineGrainedLineage +* `/urn:li:schemaField:...` -> The downstream schemaField referenced in this schema, part of the key for a fineGrainedLineage +* `/urn:li:query:...` -> The query urn this relationship was derived from, part of the key for a fineGrainedLineage +* `/urn:li:schemaField:` -> The upstream urn that is being targeted by this patch operation + +This showcases that in some cases the key for objects is simple, in others in can be complex to determine, but for our fully supported use cases we have +SDK support on both the Java and Python side that will generate these patches for you as long as you supply the required method parameters. +Path is generally the most complicated portion of a patch to reason about as it requires intimate knowledge of the schema and models. + +### Operation + +Operation is a limited enum of a few supported types pulled directly from the JSON Patch spec. DataHub only supports `ADD` and `REMOVE` of these options +as the other patch operations do not currently have a use case within our system. + +#### Add + +Add is a bit of a misnomer for the JSON Patch spec, it is not an explicit add but an upsert/replace. If the path specified does not exist, it will be created, +but if the path already exists the value will be replaced. Patch operations apply at a path level so it is possible to do full replaces of arrays or objects in the schema +using adds, but generally the most useful use case for patch is to add elements to arrays without affecting the other elements as full upserts are supported by standard ingestion. + +#### Remove + +Remove operations require the path specified to be present, or an error will be thrown, otherwise they operate as one would expect. The specified path will be removed from the aspect. + + +### Value + +Value is the actual information that will be stored at a path. If the path references an object then this will include the JSON key value pairs for that object. + +#### Examples + +An example UpstreamLineage object value: + +```json +{ + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,my-bucket/my-folder/my-file.txt,PROD)", + "type": "TRANSFORMED" +} +``` + +For the previous path example (`/upstreams/urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created_upstream,PROD)`), this object would represent the UpstreamLineage object for that path. +This specifies the required fields to properly represent that object. Note: by modifying this path, you could reference a single field within the UpstreamLineage object itself, like so: + +```json +{ + "path": "/upstreams/urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created_upstream,PROD)/type", + "op": "ADD", + "value": "VIEW" +} +``` + +### Implementation details + +#### Template Classes + +Template classes are the mechanism that maps fields to their corresponding JSON paths. Since DataMaps are not true JSON, first we convert a RecordTemplate to a JSON String, +perform any additional process to map array fields to their keys, apply the patch, and then convert the JSON object back to a RecordTemplate to work with the rest of the application. + +The template classes we currently support can be found in the `entity-registry` module. They are split up by aspect, with the GenericTemplate applying to any non-directly supported aspects. +The GenericTemplate allows for use cases that we have not gotten around to directly support yet, but puts more burden on the user to generate patches correctly. + +The template classes are utilized in `EntityServiceImpl` where a MCP is determined to be either a patch or standard upsert which then routes through to the stored templates registered on the EntityRegistry. +The core logical flow each Template runs through is set up in the `Template` interface, with some more specific logic in the lower level interfaces for constructing/deconstructing array field keys. +Most of the complexity around these classes is knowledge of schema and JSON path traversals. + +##### ArrayMergingTemplate & CompoundKeyTemplate + +`ArrayMergingTemplate` is utilized for any aspect which has array fields and may either be used directly or use `CompoundKeyTemplate`. `ArrayMergingTemplate` is the simpler one that can only be used directly for +single value keys. `CompoundKeyTemplate` allows for support of multi-field keys. For more complex examples like FineGrainedLineage, further logic is needed to construct a key as it is not generalizable to other aspects, see `UpstreamLineageTemplate` for full special case implementation. + +#### PatchBuilders + +There are patch builder SDK classes for constructing patches in both Java and Python. The Java patch builders all extend `AbstractMultiFieldPatchBuilder` which sets up the +base functionality for patch builder subtypes. Each implementation of this abstract class is targeted at a particular aspect and contains specific field based update methods +for the most common use cases. On the Python side patch builders live in the `src/specific/` directory and are organized by entity type. \ No newline at end of file diff --git a/docs/api/datahub-apis.md b/docs/api/datahub-apis.md index ed48eb0f52fa2..6bb793a59a86e 100644 --- a/docs/api/datahub-apis.md +++ b/docs/api/datahub-apis.md @@ -1,58 +1,44 @@ -# Which DataHub API is for me? +# DataHub APIs + +DataHub has several APIs to manipulate metadata on the platform. Here's the list of APIs and their pros and cons to help you choose the right one for your use case. + +| API | Definition | Pros | Cons | +|--------------------------------------------------------------------------------|------------------------------------|------------------------------------------|-------------------------------------------------------------| +| **[Python SDK](/metadata-ingestion/as-a-library.md)** | SDK | Highly flexible, Good for bulk execution | Requires an understanding of the metadata change event | +| **[Java SDK](/metadata-integration/java/as-a-library.md)** | SDK | Highly flexible, Good for bulk execution | Requires an understanding of the metadata change event | +| **[GraphQL API](docs/api/graphql/getting-started.md)** | GraphQL interface | Intuitive; mirrors UI capabilities | Less flexible than SDKs; requires knowledge of GraphQL syntax | +| **[OpenAPI](docs/api/openapi/openapi-usage-guide.md)**
(Not Recommended) | Lower-level API for advanced users | | Generally not recommended for typical use cases | + +In general, **Python and Java SDKs** are our most recommended tools for extending and customizing the behavior of your DataHub instance. +We don't recommend using the **OpenAPI** directly, as it's more complex and less user-friendly than the other APIs. -DataHub supplys several APIs to manipulate metadata on the platform. These are our most-to-least recommended approaches: -- Our most recommended tools for extending and customizing the behavior of your DataHub instance are our SDKs in [Python](metadata-ingestion/as-a-library.md) and [Java](metadata-integration/java/as-a-library.md). -- If you'd like to customize the DataHub client or roll your own; the [GraphQL API](docs/api/graphql/getting-started.md) is our what powers our frontend. We figure if it's good enough for us, it's good enough for everyone! If `graphql` doesn't cover everything in your usecase, drop into [our slack](docs/slack.md) and let us know how we can improve it! -- If you are less familiar with `graphql` and would rather use OpenAPI, we offer [OpenAPI](docs/api/openapi/openapi-usage-guide.md) endpoints that allow you to produce metadata events and query metadata. -- Finally, if you're a brave soul and know exactly what you are doing... are you sure you don't just want to use the SDK directly? If you insist, the [Rest.li API](docs/api/restli/restli-overview.md) is a much more powerful, low level API intended only for advanced users. ## Python and Java SDK -We offer an SDK for both Python and Java that provide full functionality when it comes to CRUD operations and any complex functionality you may want to build into DataHub. - -Get started with the Python SDK - +We offer an SDK for both Python and Java that provide full functionality when it comes to CRUD operations and any complex functionality you may want to build into DataHub. We recommend using the SDKs for most use cases. Here are the examples of how to use the SDKs: + +- Define a lineage between data entities +- Executing bulk operations - e.g. adding tags to multiple datasets +- Creating custom metadata entities + +Learn more about the SDKs: +- **[Python SDK →](/metadata-ingestion/as-a-library.md)** +- **[Java SDK →](/metadata-integration/java/as-a-library.md)** - -Get started with the Java SDK - ## GraphQL API The `graphql` API serves as the primary public API for the platform. It can be used to fetch and update metadata programatically in the language of your choice. Intended as a higher-level API that simplifies the most common operations. - -Get started with the GraphQL API - - -## OpenAPI - -For developers who prefer OpenAPI to GraphQL for programmatic operations. Provides lower-level API access to the entire DataHub metadata model for writes, reads and queries. - -Get started with OpenAPI - - -## Rest.li API - -:::caution -The Rest.li API is intended only for advanced users. If you're just getting started with DataHub, we recommend the GraphQL API -::: - -The Rest.li API represents the underlying persistence layer, and exposes the raw PDL models used in storage. Under the hood, it powers the GraphQL API. Aside from that, it is also used for system-specific ingestion of metadata, being used by the Metadata Ingestion Framework for pushing metadata into DataHub directly. For all intents and purposes, the Rest.li API is considered system-internal, meaning DataHub components are the only ones to consume this API directly. - -Get started with our Rest.li API - +We recommend using the GraphQL API if you're getting started with DataHub since it's more user-friendly and straighfowrad. Here are some examples of how to use the GraphQL API: +- Search for datasets with conditions +- Update a certain field of a dataset + +Learn more about the GraphQL API: +- **[GraphQL API →](docs/api/graphql/getting-started.md)** + + ## DataHub API Comparison diff --git a/docs/api/openapi/openapi-usage-guide.md b/docs/api/openapi/openapi-usage-guide.md index f33c20c91dacb..59a331b724cfa 100644 --- a/docs/api/openapi/openapi-usage-guide.md +++ b/docs/api/openapi/openapi-usage-guide.md @@ -119,7 +119,7 @@ curl --location --request POST 'localhost:8080/openapi/entities/v1/' \ The second POST example will write the update ONLY if the entity doesn't exist. If the entity does exist the command will return an error instead of overwriting the entity. -In this example we've added an additional URL parameter `createEntityIfNotExists=true` +In this example we've added a URL parameter `createEntityIfNotExists=true` ```shell curl --location --request POST 'localhost:8080/openapi/entities/v1/?createEntityIfNotExists=true' \ @@ -582,3 +582,232 @@ public class Main { } } ``` + +## OpenAPI v3 Features + +### Conditional Writes + +All the create/POST endpoints for aspects support `headers` in the POST body to support batch APIs. See the docs in the +[MetadataChangeProposal](../../advanced/mcp-mcl.md) section for the use of these headers to support conditional writes semantics. + +### Batch Get + +Batch get endpoints in the form of `/v3/entity/{entityName}/batchGet` exist for all entities. This endpoint allows +fetching entity and aspects in bulk. In combination with the `If-Version-Match` header it can also retrieve +a specific version of the aspects, however it defaults to the latest aspect version. Currently, this interface is limited +to returning a single version for each entity/aspect however different versions can be specified across entities. + +A few example queries are as follows: + +Example Request: + +Fetch the latest aspects for the given URNs with the url parameter `systemMetadata=true` in order to view the current +versions of the aspects. + +```json +[ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)", + "globalTags": {}, + "datasetProperties": {} + }, + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "globalTags": {}, + "datasetProperties": {} + } +] +``` + +Example Response: + +Notice that `systemMetadata` contains `"version": "1"` for each of the aspects that exist in the system. + +```json +[ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)", + "datasetProperties": { + "value": { + "description": "table containing all the users deleted on a single day", + "customProperties": { + "encoding": "utf-8" + }, + "tags": [] + }, + "systemMetadata": { + "properties": { + "clientVersion": "1!0.0.0.dev0", + "clientId": "acryl-datahub" + }, + "version": "1", + "lastObserved": 1720781548776, + "lastRunId": "file-2024_07_12-05_52_28", + "runId": "file-2024_07_12-05_52_28" + } + } + }, + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "datasetProperties": { + "value": { + "description": "table containing all the users created on a single day", + "customProperties": { + "encoding": "utf-8" + }, + "tags": [] + }, + "systemMetadata": { + "properties": { + "clientVersion": "1!0.0.0.dev0", + "clientId": "acryl-datahub" + }, + "version": "1", + "lastObserved": 1720781548773, + "lastRunId": "file-2024_07_12-05_52_28", + "runId": "file-2024_07_12-05_52_28" + } + }, + "globalTags": { + "value": { + "tags": [ + { + "tag": "urn:li:tag:NeedsDocumentation" + } + ] + }, + "systemMetadata": { + "properties": { + "appSource": "ui" + }, + "version": "1", + "lastObserved": 0, + "lastRunId": "no-run-id-provided", + "runId": "no-run-id-provided" + } + } + } +] +``` + +Next let's mutate `globalTags` for the second URN by adding a new tag. This will increment the version of +the `globalTags` aspect. The response will then look at like the following, notice the incremented +`"version": "2"` in `systemMetadata` for the `globalTags` aspect. Also notice that there are now 2 tags present, unlike +previously where only `urn:li:tag:NeedsDocumentation` was present. + +```json +[ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)", + "datasetProperties": { + "value": { + "description": "table containing all the users deleted on a single day", + "customProperties": { + "encoding": "utf-8" + }, + "tags": [] + }, + "systemMetadata": { + "properties": { + "clientVersion": "1!0.0.0.dev0", + "clientId": "acryl-datahub" + }, + "version": "1", + "lastObserved": 1720781548776, + "lastRunId": "file-2024_07_12-05_52_28", + "runId": "file-2024_07_12-05_52_28" + } + } + }, + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "datasetProperties": { + "value": { + "description": "table containing all the users created on a single day", + "customProperties": { + "encoding": "utf-8" + }, + "tags": [] + }, + "systemMetadata": { + "properties": { + "clientVersion": "1!0.0.0.dev0", + "clientId": "acryl-datahub" + }, + "version": "1", + "lastObserved": 1720781548773, + "lastRunId": "file-2024_07_12-05_52_28", + "runId": "file-2024_07_12-05_52_28" + } + }, + "globalTags": { + "value": { + "tags": [ + { + "tag": "urn:li:tag:NeedsDocumentation" + }, + { + "tag": "urn:li:tag:Legacy" + } + ] + }, + "systemMetadata": { + "properties": { + "appSource": "ui" + }, + "version": "2", + "lastObserved": 0, + "lastRunId": "no-run-id-provided", + "runId": "no-run-id-provided" + } + } + } +] +``` + +Next, we'll retrieve the previous version of the `globalTags` for the one aspect with a version 2 with the following query. +We can do this by populating the `headers` map with `If-Version-Match` to retrieve the previous version 1. + +Example Request: +```json +[ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "globalTags": { + "headers": { + "If-Version-Match": "1" + } + } + } +] +``` + +Example Response: + +The previous version `1` of the `globalTags` aspect is returned as expected with only the single tag. + +```json +[ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "globalTags": { + "value": { + "tags": [ + { + "tag": "urn:li:tag:NeedsDocumentation" + } + ] + }, + "systemMetadata": { + "properties": { + "appSource": "ui" + }, + "version": "1", + "lastObserved": 0, + "lastRunId": "no-run-id-provided", + "runId": "no-run-id-provided" + } + } + } +] +``` diff --git a/docs/api/tutorials/assertions.md b/docs/api/tutorials/assertions.md new file mode 100644 index 0000000000000..f89fe728f7e97 --- /dev/null +++ b/docs/api/tutorials/assertions.md @@ -0,0 +1,1183 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Assertions + + + +This guide specifically covers how to use the Assertion APIs for **Acryl Cloud** native assertions, including: + +- [Freshness Assertions](/docs/managed-datahub/observe/freshness-assertions.md) +- [Volume Assertions](/docs/managed-datahub/observe/volume-assertions.md) +- [Column Assertions](/docs/managed-datahub/observe/column-assertions.md) +- [Schema Assertions](/docs/managed-datahub/observe/schema-assertions.md) +- [Custom SQL Assertions](/docs/managed-datahub/observe/custom-sql-assertions.md) + +## Why Would You Use Assertions APIs? + +The Assertions APIs allow you to create, schedule, run, and delete Assertions with Acryl Cloud. + +### Goal Of This Guide + +This guide will show you how to create, schedule, run and delete Assertions for a Table. + +## Prerequisites + +The actor making API calls must have the `Edit Assertions` and `Edit Monitors` privileges for the Tables at hand. + +## Create Assertions + +You can create new dataset Assertions to DataHub using the following APIs. + + + + +### Freshness Assertion + +To create a new freshness assertion, use the `upsertDatasetFreshnessAssertionMonitor` GraphQL Mutation. + +```graphql +mutation upsertDatasetFreshnessAssertionMonitor { + upsertDatasetFreshnessAssertionMonitor( + input: { + entityUrn: "", + schedule: { + type: FIXED_INTERVAL, + fixedInterval: { unit: HOUR, multiple: 8 } + } + evaluationSchedule: { + timezone: "America/Los_Angeles", + cron: "0 */8 * * *" + } + evaluationParameters: { + sourceType: INFORMATION_SCHEMA + } + mode: ACTIVE + } + ) { + urn + } +} +``` + +This API will return a unique identifier (URN) for the new assertion if you were successful: + +```json +{ + "data": { + "upsertDatasetFreshnessAssertionMonitor": { + "urn": "urn:li:assertion:your-new-assertion-id" + } + }, + "extensions": {} +} +``` + +For more details, see the [Freshness Assertions](/docs/managed-datahub/observe/freshness-assertions.md) guide. + +### Volume Assertions + +To create a new volume assertion, use the `upsertDatasetVolumeAssertionMonitor` GraphQL Mutation. + +```graphql +mutation upsertDatasetVolumeAssertionMonitor { + upsertDatasetVolumeAssertionMonitor( + input: { + entityUrn: "" + type: ROW_COUNT_TOTAL + rowCountTotal: { + operator: BETWEEN + parameters: { + minValue: { + value: "10" + type: NUMBER + } + maxValue: { + value: "20" + type: NUMBER + } + } + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + evaluationParameters: { + sourceType: INFORMATION_SCHEMA + } + mode: ACTIVE + } + ) { + urn + } +} +``` + +This API will return a unique identifier (URN) for the new assertion if you were successful: + +```json +{ + "data": { + "upsertDatasetVolumeAssertionMonitor": { + "urn": "urn:li:assertion:your-new-assertion-id" + } + }, + "extensions": {} +} +``` + +For more details, see the [Volume Assertions](/docs/managed-datahub/observe/volume-assertions.md) guide. + +### Column Assertions + +To create a new column assertion, use the `upsertDatasetFieldAssertionMonitor` GraphQL Mutation. + +```graphql +mutation upsertDatasetFieldAssertionMonitor { + upsertDatasetFieldAssertionMonitor( + input: { + entityUrn: "" + type: FIELD_VALUES, + fieldValuesAssertion: { + field: { + path: "", + type: "NUMBER", + nativeType: "NUMBER(38,0)" + }, + operator: GREATER_THAN, + parameters: { + value: { + type: NUMBER, + value: "10" + } + }, + failThreshold: { + type: COUNT, + value: 0 + }, + excludeNulls: true + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + evaluationParameters: { + sourceType: ALL_ROWS_QUERY + } + mode: ACTIVE + } + ){ + urn + } +} +``` + +This API will return a unique identifier (URN) for the new assertion if you were successful: + +```json +{ + "data": { + "upsertDatasetFieldAssertionMonitor": { + "urn": "urn:li:assertion:your-new-assertion-id" + } + }, + "extensions": {} +} +``` + +For more details, see the [Column Assertions](/docs/managed-datahub/observe/column-assertions.md) guide. + +### Custom SQL Assertions + +To create a new column assertion, use the `upsertDatasetSqlAssertionMonitor` GraphQL Mutation. + +```graphql +mutation upsertDatasetSqlAssertionMonitor { + upsertDatasetSqlAssertionMonitor( + assertionUrn: "" + input: { + entityUrn: "" + type: METRIC, + description: "", + statement: "", + operator: GREATER_THAN_OR_EQUAL_TO, + parameters: { + value: { + value: "100", + type: NUMBER + } + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */6 * * *" + } + mode: ACTIVE + } + ) { + urn + } +} +``` + +This API will return a unique identifier (URN) for the new assertion if you were successful: + +```json +{ + "data": { + "upsertDatasetSqlAssertionMonitor": { + "urn": "urn:li:assertion:your-new-assertion-id" + } + }, + "extensions": {} +} +``` + +For more details, see the [Custom SQL Assertions](/docs/managed-datahub/observe/custom-sql-assertions.md) guide. + +### Schema Assertions + +To create a new schema assertion, use the `upsertDatasetSchemaAssertionMonitor` GraphQL Mutation. + +```graphql +mutation upsertDatasetSchemaAssertionMonitor { + upsertDatasetSchemaAssertionMonitor( + assertionUrn: "urn:li:assertion:existing-assertion-id", + input: { + entityUrn: "", + assertion: { + compatibility: EXACT_MATCH, + fields: [ + { + path: "id", + type: STRING + }, + { + path: "count", + type: NUMBER + }, + { + path: "struct", + type: STRUCT + }, + { + path: "struct.nestedBooleanField", + type: BOOLEAN + } + ] + }, + description: "", + mode: ACTIVE + } + ) +} +``` + +This API will return a unique identifier (URN) for the new assertion if you were successful: + +```json +{ + "data": { + "upsertDatasetSchemaAssertionMonitor": { + "urn": "urn:li:assertion:your-new-assertion-id" + } + }, + "extensions": {} +} +``` + +For more details, see the [Schema Assertions](/docs/managed-datahub/observe/schema-assertions.md) guide. + + + + + +## Run Assertions + +You can use the following APIs to trigger the assertions you've created to run on-demand. This is +particularly useful for running assertions on a custom schedule, for example from your production +data pipelines. + +> **Long-Running Assertions**: The timeout for synchronously running an assertion is currently limited to a maximum of 30 seconds. +> Each of the following APIs support an `async` parameter, which can be set to `true` to run the assertion asynchronously. +> When set to `true`, the API will kick off the assertion run and return null immediately. To view the result of the assertion, +> simply fetching the runEvents field of the `assertion(urn: String!)` GraphQL query. + + + + +### Run Assertion + +```graphql +mutation runAssertion { + runAssertion(urn: "urn:li:assertion:your-assertion-id", saveResult: true) { + type + nativeResults { + key + value + } + } +} +``` + +Where **type** will contain the Result of the assertion run, either `SUCCESS`, `FAILURE`, or `ERROR`. + +The `saveResult` argument determines whether the result of the assertion will be saved to DataHub's backend, +and available to view through the DataHub UI. If this is set to false, the result will NOT be stored in DataHub's +backend. The value defaults to `true`. + +If the assertion is external (not natively executed by Acryl), this API will return an error. + +If running the assertion is successful, the result will be returned as follows: + +```json +{ + "data": { + "runAssertion": { + "type": "SUCCESS", + "nativeResults": [ + { + "key": "Value", + "value": "1382" + } + ] + } + }, + "extensions": {} +} +``` + +### Run Group of Assertions + +```graphql +mutation runAssertions { + runAssertions(urns: ["urn:li:assertion:your-assertion-id-1", "urn:li:assertion:your-assertion-id-2"], saveResults: true) { + passingCount + failingCount + errorCount + results { + urn + result { + type + nativeResults { + key + value + } + } + } + } +} +``` + +Where **type** will contain the Result of the assertion run, either `SUCCESS`, `FAILURE`, or `ERROR`. + +The `saveResults` argument determines whether the result of the assertion will be saved to DataHub's backend, +and available to view through the DataHub UI. If this is set to false, the result will NOT be stored in DataHub's +backend. The value defaults to `true`. + +If any of the assertion are external (not natively executed by Acryl), they will simply be omitted from the result set. + +If running the assertions is successful, the results will be returned as follows: + +```json +{ + "data": { + "runAssertions": { + "passingCount": 2, + "failingCount": 0, + "errorCount": 0, + "results": [ + { + "urn": "urn:li:assertion:your-assertion-id-1", + "result": { + "type": "SUCCESS", + "nativeResults": [ + { + "key": "Value", + "value": "1382" + } + ] + } + }, + { + "urn": "urn:li:assertion:your-assertion-id-2", + "result": { + "type": "FAILURE", + "nativeResults": [ + { + "key": "Value", + "value": "12323" + } + ] + } + } + ] + } + }, + "extensions": {} +} +``` + +Where you should see one result object for each assertion. + +### Run All Assertions for Table + +You can also run all assertions for a specific data asset using the `runAssertionsForAsset` mutation. + +```graphql +mutation runAssertionsForAsset { + runAssertionsForAsset(urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,purchase_events,PROD)", saveResults: true) { + passingCount + failingCount + errorCount + results { + urn + result { + type + nativeResults { + key + value + } + } + } + } +} +``` + +Where `type` will contain the Result of the assertion run, either `SUCCESS`, `FAILURE`, or `ERROR`. + +The `saveResults` argument determines whether the result of the assertion will be saved to DataHub's backend, +and available to view through the DataHub UI. If this is set to false, the result will NOT be stored in DataHub's +backend. The value defaults to `true`. + +If any of the assertion are external (not natively executed by Acryl), they will simply be omitted from the result +set. + +If running the assertions is successful, the results will be returned as follows: + +```json +{ + "data": { + "runAssertionsForAsset": { + "passingCount": 2, + "failingCount": 0, + "errorCount": 0, + "results": [ + { + "urn": "urn:li:assertion:your-assertion-id-1", + "result": { + "type": "SUCCESS", + "nativeResults": [ + { + "key": "Value", + "value": "1382" + } + ] + } + }, + { + "urn": "urn:li:assertion:your-assertion-id-2", + "result": { + "type": "FAILURE", + "nativeResults": [ + { + "key": "Value", + "value": "12323" + } + ] + } + } + ] + } + }, + "extensions": {} +} +``` + +Where you should see one result object for each assertion. + +### Run Group of Assertions for Table + +If you don't always want to run _all_ assertions for a given table, you can also opt to run a subset of the +table's assertions using *Assertion Tags*. First, you'll add tags to your assertions to group and categorize them, +then you'll call the `runAssertionsForAsset` mutation with the `tagUrns` argument to filter for assertions having those tags. + +#### Step 1: Adding Tag to an Assertion + +Currently, you can add tags to an assertion only via the DataHub GraphQL API. You can do this using the following mutation: + +```graphql +mutation addTags { + addTag(input: { + resourceUrn: "urn:li:assertion:your-assertion", + tagUrn: "urn:li:tag:my-important-tag", + }) +} +``` + +#### Step 2: Run All Assertions for a Table with Tags + +Now, you can run all assertions for a table with a specific tag(s) using the `runAssertionsForAsset` mutation with the +`tagUrns` input parameter: + +```graphql +mutation runAssertionsForAsset { + runAssertionsForAsset(urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,purchase_events,PROD)", tagUrns: ["urn:li:tag:my-important-tag"]) { + passingCount + failingCount + errorCount + results { + urn + result { + type + nativeResults { + key + value + } + } + } + } +} +``` + +**Coming Soon**: Support for adding tags to assertions through the DataHub UI. + + + + + +### Run Assertion + +```python +{{ inline /metadata-ingestion/examples/library/run_assertion.py show_path_as_comment }} +``` + +### Run Group of Assertions + +```python +{{ inline /metadata-ingestion/examples/library/run_assertions.py show_path_as_comment }} +``` + +### Run All Assertions for Table + +```python +{{ inline /metadata-ingestion/examples/library/run_assertions_for_asset.py show_path_as_comment }} +``` + + + + + +### Experimental: Providing Dynamic Parameters to Assertions + +You can provide **dynamic parameters** to your assertions to customize their behavior. This is particularly useful for +assertions that require dynamic parameters, such as a threshold value that changes based on the time of day. + +Dynamic parameters can be injected into the SQL fragment portion of any Assertion. For example, it can appear +in any part of the SQL statement in a [Custom SQL](/docs/managed-datahub/observe/custom-sql-assertions.md) Assertion, +or it can appear in the **Advanced > Filter** section of a [Column](/docs/managed-datahub/observe/column-assertions.md), +[Volume](/docs/managed-datahub/observe/volume-assertions.md), or [Freshness](/docs/managed-datahub/observe/freshness-assertions.md) Assertion. + +To do so, you'll first need to edit the SQL fragment to include the dynamic parameter. Dynamic parameters appear +as `${parameterName}` in the SQL fragment. + +Next, you'll call the `runAssertion`, `runAssertions`, or `runAssertionsForAsset` mutations with the `parameters` input argument. +This argument is a list of key-value tuples, where the key is the parameter name and the value is the parameter value: + +```graphql +mutation runAssertion { + runAssertion(urn: "urn:li:assertion:your-assertion-id", parameters: [{key: "parameterName", value: "parameterValue"}]) { + type + nativeResults { + key + value + } + } +} +``` + +At runtime, the `${parameterName}` placeholder in the SQL fragment will be replaced with the provided `parameterValue` before the query +is sent to the database for execution. + +## Get Assertion Details + +You can use the following APIs to + +1. Fetch existing assertion definitions + run history +2. Fetch the assertions associated with a given table + their run history. + + + + +### Get Assertions for Table + +To retrieve all the assertions for a table, you can use the following GraphQL Query. + +```graphql +query dataset { + dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,purchases,PROD)") { + assertions(start: 0, count: 1000) { + start + count + total + assertions { + urn + # Fetch the last run of each associated assertion. + runEvents(status: COMPLETE, limit: 1) { + total + failed + succeeded + runEvents { + timestampMillis + status + result { + type + nativeResults { + key + value + } + } + } + } + info { + type + description + lastUpdated { + time + actor + } + datasetAssertion { + datasetUrn + scope + aggregation + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + fields { + urn + path + } + nativeType + nativeParameters { + key + value + } + logic + } + freshnessAssertion { + type + entityUrn + schedule { + type + cron { + cron + timezone + } + fixedInterval { + unit + multiple + } + } + filter { + type + sql + } + } + sqlAssertion { + type + entityUrn + statement + changeType + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + fieldAssertion { + type + entityUrn + filter { + type + sql + } + fieldValuesAssertion { + field { + path + type + nativeType + } + transform { + type + } + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + failThreshold { + type + value + } + excludeNulls + } + fieldMetricAssertion { + field { + path + type + nativeType + } + metric + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + } + volumeAssertion { + type + entityUrn + filter { + type + sql + } + rowCountTotal { + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + rowCountChange { + type + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + } + schemaAssertion { + entityUrn + compatibility + fields { + path + type + nativeType + } + schema { + fields { + fieldPath + type + nativeDataType + } + } + } + source { + type + created { + time + actor + } + } + } + } + } + } +} +``` + +### Get Assertion Details + +You can use the following GraphQL query to fetch the details for an assertion along with its evaluation history by URN. + +```graphql +query getAssertion { + assertion(urn: "urn:li:assertion:assertion-id") { + urn + # Fetch the last 10 runs for the assertion. + runEvents(status: COMPLETE, limit: 10) { + total + failed + succeeded + runEvents { + timestampMillis + status + result { + type + nativeResults { + key + value + } + } + } + } + info { + type + description + lastUpdated { + time + actor + } + datasetAssertion { + datasetUrn + scope + aggregation + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + fields { + urn + path + } + nativeType + nativeParameters { + key + value + } + logic + } + freshnessAssertion { + type + entityUrn + schedule { + type + cron { + cron + timezone + } + fixedInterval { + unit + multiple + } + } + filter { + type + sql + } + } + sqlAssertion { + type + entityUrn + statement + changeType + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + fieldAssertion { + type + entityUrn + filter { + type + sql + } + fieldValuesAssertion { + field { + path + type + nativeType + } + transform { + type + } + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + failThreshold { + type + value + } + excludeNulls + } + fieldMetricAssertion { + field { + path + type + nativeType + } + metric + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + } + volumeAssertion { + type + entityUrn + filter { + type + sql + } + rowCountTotal { + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + rowCountChange { + type + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + } + schemaAssertion { + entityUrn + compatibility + fields { + path + type + nativeType + } + schema { + fields { + fieldPath + type + nativeDataType + } + } + } + source { + type + created { + time + actor + } + } + } + } +} +``` + + + + + +```python +Python support coming soon! +``` + + + + +## Add Tag to Assertion + +You can add tags to individual assertions to group and categorize them, for example by its priority or severity. +Note that the tag should already exist in DataHub, or the operation will fail. + + + + +```graphql +mutation addTags { + addTag(input: { + resourceUrn: "urn:li:assertion:your-assertion", + tagUrn: "urn:li:tag:my-important-tag", + }) +} +``` + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "addTag": true + }, + "extensions": {} +} +``` + +You can create new tags using the `createTag` mutation or via the UI. + + + + +## Delete Assertions + +You can use delete dataset operations to DataHub using the following APIs. + + + + +```graphql +mutation deleteAssertion { + deleteAssertion(urn: "urn:li:assertion:test") +} +``` + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "deleteAssertion": true + }, + "extensions": {} +} +``` + + + + + +```python +{{ inline /metadata-ingestion/examples/library/delete_assertion.py show_path_as_comment }} +``` + + + + +## (Advanced) Create and Report Results for Custom Assertions + +If you'd like to create and report results for your own custom assertions, e.g. those which are run and +evaluated outside of Acryl, you need to generate 2 important Assertion Entity aspects, and give the assertion a unique +URN of the following format: + + +1. Generate a unique URN for your assertion + +```plaintext +urn:li:assertion: +``` + +2. Generate the [**AssertionInfo**](/docs/generated/metamodel/entities/assertion.md#assertion-info) aspect for the assertion. You can do this using the Python SDK. Give your assertion a `type` and a `source` +with type `EXTERNAL` to mark it as an external assertion, not run by DataHub itself. + +3. Generate the [**AssertionRunEvent**](/docs/generated/metamodel/entities/assertion.md#assertionrunevent-timeseries) timeseries aspect using the Python SDK. This aspect should contain the result of the assertion +run at a given timestamp and will be shown on the results graph in DataHub's UI. + diff --git a/docs/api/tutorials/custom-assertions.md b/docs/api/tutorials/custom-assertions.md new file mode 100644 index 0000000000000..6544efb8809c2 --- /dev/null +++ b/docs/api/tutorials/custom-assertions.md @@ -0,0 +1,290 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Custom Assertions + +This guide specifically covers how to create and report results for custom assertions in DataHub. +Custom Assertions are those not natively run or directly modeled by DataHub, and managed by a 3rd party framework or tool. + +To create _native_ assertions using the API (e.g. for DataHub to manage), please refer to the [Assertions API](./assertions.md). + +This guide may be used as reference for partners seeking to integrate their own monitoring tools with DataHub. + +## Goal Of This Guide + +In this guide, you will learn how to + +1. Create and update custom assertions via GraphQL and Python APIs +2. Report results for custom assertions via GraphQL and Python APIs +3. Retrieve results for custom assertions via GraphQL and Python APIs +4. Delete custom assertions via GraphQL and Python APIs + +## Prerequisites + +The actor making API calls must have the `Edit Assertions` and `Edit Monitors` privileges for the Tables being monitored. + +## Create And Update Custom Assertions + +You may create custom assertions using the following APIs for a Dataset in DataHub. + + + + +To create a new assertion, use the `upsertCustomAssertion` GraphQL Mutation. This mutation both allows you to +create and update a given assertion. + +```graphql +mutation upsertCustomAssertion { + upsertCustomAssertion( + urn: "urn:li:assertion:my-custom-assertion-id", # Optional: if you want to provide a custom id. If not, one will be generated for you. + input: { + entityUrn: "", + type: "My Custom Category", # This is how your assertion will appear categorized in DataHub. + description: "The description of my external assertion for my dataset", + platform: { + urn: "urn:li:dataPlatform:great-expectations", # OR you can provide name: "My Custom Platform" if you do not have an URN for the platform. + } + fieldPath: "field_foo", # Optional: if you want to associated with a specific field, + externalUrl: "https://my-monitoring-tool.com/result-for-this-assertion" # Optional: if you want to provide a link to the monitoring tool + # Optional: If you want to provide a custom SQL query for the assertion. This will be rendered as a query in the UI. + # logic: "SELECT * FROM X WHERE Y" + } + ) { + urn + } +} +``` + +Note that you can either provide a unique `urn` for the assertion, which will be used to generate the corresponding assertion urn in the following format: + +`urn:li:assertion:` + +or a random urn will be created and returned for you. This id should be stable over time and unique for each assertion. + +The upsert API will return the unique identifier (URN) for the the assertion if you were successful: + +```json +{ + "data": { + "upsertExternalAssertion": { + "urn": "urn:li:assertion:your-new-assertion-id" + } + }, + "extensions": {} +} +``` + + + + + +To upsert an assertion in Python, simply use the `upsert_external_assertion` method on the DataHub Client object. + +```python +{{ inline /metadata-ingestion/examples/library/upsert_custom_assertion.py show_path_as_comment }} +``` + + + + + +## Report Results For Custom Assertions + +When an assertion is evaluated against a Dataset, or a new result is available, you can report the result to DataHub +using the following APIs. + +Once reported, these will appear in the evaluation history of the assertion and will be used to determine whether the assertion is +displayed as passing or failing in the DataHub UI. + + + + +To report results for a custom, use the `reportAssertionResult` GraphQL Mutation. This mutation both allows you to +create and update a given assertion. + +```graphql +mutation reportAssertionResult { + reportAssertionResult( + urn: "urn:li:assertion:" + result: { + timestampMillis: 1620000000, # Unix timestamp in millis. If not provided, the current time will be used. + type: SUCCESS, # or FAILURE or ERROR or INIT + properties: [ + { + key: "my_custom_key", + value: "my_custom_value" + } + ], + externalUrl: "https://my-great-expectations.com/results/1234", # Optional: URL to the results in the external tool + # Optional: If the type is ERROR, you can provide additional context. See full list of error types below. + # error: { + # type: UNKNOWN_ERROR, + # message: "The assertion failed due to an unknown error" + # } + } + ) +} +``` + +The `type` field is used to communicate the latest health status of the assertion. + +The `properties` field is used to provide additional key-value pair context that will be displayed alongside the result +in DataHub's UI. + +The full list of supported error types include: + +- SOURCE_CONNECTION_ERROR +- SOURCE_QUERY_FAILED +- INSUFFICIENT_DATA +- INVALID_PARAMETERS +- INVALID_SOURCE_TYPE +- UNSUPPORTED_PLATFORM +- CUSTOM_SQL_ERROR +- FIELD_ASSERTION_ERROR +- UNKNOWN_ERROR + +```json +{ + "data": { + "reportAssertionResult": true + }, + "extensions": {} +} +``` + +If the result is `true`, the result was successfully reported. + + + + + +To report an assertion result in Python, simply use the `report_assertion_result` method on the DataHub Client object. + +```python +{{ inline /metadata-ingestion/examples/library/report_assertion_result.py show_path_as_comment }} +``` + + + + + +## Retrieve Results For Custom Assertions + +After an assertion has been created and run, it will appear in the set of assertions associated with a given dataset urn. +You can retrieve the results of these assertions using the following APIs. + + + + + +### Get Assertions for Dataset + +To retrieve all the assertions for a table / dataset, you can use the following GraphQL Query. + +```graphql +query dataset { + dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,purchases,PROD)") { + assertions(start: 0, count: 1000) { + start + count + total + assertions { + urn + # Fetch the last run of each associated assertion. + runEvents(status: COMPLETE, limit: 1) { + total + failed + succeeded + runEvents { + timestampMillis + status + result { + type + nativeResults { + key + value + } + } + } + } + info { + type # Will be CUSTOM + customType # Will be your custom type. + description + lastUpdated { + time + actor + } + customAssertion { + entityUrn + fieldPath + externalUrl + logic + } + source { + type + created { + time + actor + } + } + } + } + } + } +} +``` + +### Get Assertion Details + +You can use the following GraphQL query to fetch the details for an assertion along with its evaluation history by URN. + +```graphql +query getAssertion { + assertion(urn: "urn:li:assertion:my-custom-assertion-id") { + urn + # Fetch the last 10 runs for the assertion. + runEvents(status: COMPLETE, limit: 10) { + total + failed + succeeded + runEvents { + timestampMillis + status + result { + type + nativeResults { + key + value + } + } + } + } + info { + type # Will be CUSTOM + customType # Will be your custom type. + description + lastUpdated { + time + actor + } + customAssertion { + entityUrn + fieldPath + externalUrl + logic + } + source { + type + created { + time + actor + } + } + } + } +} +``` + + + diff --git a/docs/api/tutorials/data-contracts.md b/docs/api/tutorials/data-contracts.md new file mode 100644 index 0000000000000..ac19920a5c4b7 --- /dev/null +++ b/docs/api/tutorials/data-contracts.md @@ -0,0 +1,217 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Data Contracts + + + +This guide specifically covers how to use the Data Contract APIs with **Acryl Cloud**. + +## Why Would You Use Data Contract APIs? + +The Assertions APIs allow you to create, update, and evaluate Data Contracts programmatically. This is particularly +useful to automate the monitoring of data quality and schema compliance for your data. + +### Goal Of This Guide + +This guide will show you how to create, update, and check the status of aData Contract. + +## Prerequisites + +### Privileges Required + +The actor making API calls must have the `Edit Data Contract` privileges for the Tables at hand. + +### Assertions + +Before creating a Data Contract, you should have already created the Assertions that you want to associate with the Data Contract. +Check out the [Assertions](/docs/api/tutorials/assertions.md) guide for details on how to create DataHub Assertions. + +## Create & Update Data Contract + +You can create a new Data Contract, which is simply bundle of "important" assertions, using the following APIs. + + + + +To create or update a Data Contract, simply use the `upsertDataContract` GraphQL Mutation. + +```graphql +mutation upsertDataContract { + upsertDataContract( + input: { + entityUrn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,purchases,PROD)", # Table to Create Contract for + freshness: [ + { + assertionUrn: "urn:li:assertion:your-freshness-assertion-id", + } + ], + schema: [ + { + assertionUrn: "urn:li:assertion:your-schema-assertion-id", + } + ], + dataQuality: [ + { + assertionUrn: "urn:li:assertion:your-column-assertion-id-1", + }, + { + assertionUrn: "urn:li:assertion:your-column-assertion-id-2", + } + ] + }) { + urn + } + ) +} +``` + +This API will return a unique identifier (URN) for the Data Contract if you were successful: + +```json +{ + "data": { + "upsertDataContract": { + "urn": "urn:li:dataContract:your-new-contract-id" + } + }, + "extensions": {} +} +``` + +If you want to update an existing Data Contract, you can use the same API, but also passing the `urn` parameter in the +`upsertDataContract` mutation. + +```graphql +mutation upsertDataContract { + upsertDataContract( + urn: "urn:li:dataContract:your-existing-contract-id", + input: { + freshness: [ + { + assertionUrn: "urn:li:assertion:your-freshness-assertion-id", + } + ], + schema: [ + { + assertionUrn: "urn:li:assertion:your-schema-assertion-id", + } + ], + dataQuality: [ + { + assertionUrn: "urn:li:assertion:your-column-assertion-id-1", + }, + { + assertionUrn: "urn:li:assertion:your-column-assertion-id-2", + } + ] + }) { + urn + } + ) +} +``` + + + + +## Check Contract Status + +You can use the following APIs to check whether a Data Contract is passing or failing, which is determined +by the last status of the assertions associated with the contract. + + + + + +### Check Contract Status for Table + +```graphql +query getTableContractStatus { + dataset(urn: "urn:li:dataset(urn:li:dataPlatform:snowflake,purchases,PROD") { + contract { + result { + type # Passing or Failing. + assertionResults { # Results of each contract assertion. + assertion { + urn + } + result { + type + nativeResults { + key + value + } + } + } + } + } + } +} +``` + +You can also _force refresh_ all of the Contract Assertions by evaluating them on-demand by providing the `refresh` argument +in your query. + +```graphql +query getTableContractStatus { + dataset(urn: "urn:li:dataset(urn:li:dataPlatform:snowflake,purchases,PROD") { + contract(refresh: true) { + ... same + } + } +} +``` + +This will run any native Acryl assertions comprising the Data Contract. Be careful! This can take a while depending on how many native assertions are part of the contract. + +If you're successful, you'll get the latest status for the Table Contract: + +```json +{ + "data": { + "dataset": { + "contract": { + "result": { + "type": "PASSING", + "assertionResults": [ + { + "assertion": { + "urn": "urn:li:assertion:your-freshness-assertion-id" + }, + "result": { + "type": "SUCCESS", + "nativeResults": [ + { + "key": "Value", + "value": "1382" + } + ] + } + }, + { + "assertion": { + "urn": "urn:li:assertion:your-volume-assertion-id" + }, + "result": { + "type": "SUCCESS", + "nativeResults": [ + { + "key": "Value", + "value": "12323" + } + ] + } + } + ] + } + } + } + }, + "extensions": {} +} +``` + + + + diff --git a/docs/api/tutorials/incidents.md b/docs/api/tutorials/incidents.md new file mode 100644 index 0000000000000..20a24d58a1db4 --- /dev/null +++ b/docs/api/tutorials/incidents.md @@ -0,0 +1,164 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Incidents + +## Why Would You Use Incidents APIs? + +The Incidents APIs allow you to raise, retrieve, update and resolve data incidents via API. This is +useful for raising or resolving data incidents programmatically, for example from Airflow, Prefect, or Dagster DAGs. +Incidents are also useful for conditional Circuit Breaking in these pipelines. + +### Goal Of This Guide + +This guide will show you how to raise, retrieve, update and resolve data incidents via API. + +## Prerequisites + +The actor making API calls must have the `Edit Incidents` privileges for the Tables at hand. + +## Raise Incident + +You can raise a new Data Incident for an existing asset using the following APIs. + + + + +```graphql +mutation raiseIncident { + raiseIncident( + input: { + resourceUrn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,public.prod.purchases,PROD)", + type: OPERATIONAL, + title: "Data is Delayed", + description: "Data is delayed on May 15, 2024 because of downtime in the Spark Cluster.", + } + ) +} +``` + +Where `resourceUrn` is the unique identifier for the data asset (dataset, dashboard, chart, data job, or data flow) you want to raise the incident on. + +Where supported Incident Types include + +- `OPERATIONAL` +- `FRESHNESS` +- `VOLUME` +- `COLUMN` +- `SQL` +- `DATA_SCHEMA` +- `CUSTOM` + +If you see the following response, a unique identifier for the new incident will be returned. + +```json +{ + "data": { + "raiseIncident": "urn:li:incident:new-incident-id" + }, + "extensions": {} +} +``` + + + + + +``` +Python SDK support coming soon! +``` + + + + + +## Get Incidents For Data Asset + +You can use retrieve the incidents and their statuses for a given Data Asset using the following APIs. + + + + +```graphql +query getAssetIncidents { + dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,public.prod.purchases,PROD)") { + incidents( + state: ACTIVE, start: 0, count: 20 + ) { + start + count + total + incidents { + urn + incidentType + title + description + status { + state + lastUpdated { + time + actor + } + } + } + } + } +} +``` + +Where you can filter for active incidents by passing the `ACTIVE` state and resolved incidents by passing the `RESOLVED` state. +This will return all relevant incidents for the dataset. + + + + + +``` +Python SDK support coming soon! +``` + + + + + +## Resolve Incidents + +You can update the status of an incident using the following APIs. + + + + +```graphql +mutation updateIncidentStatus { + updateIncidentStatus( + input: { + state: RESOLVED, + message: "The delayed data issue was resolved at 4:55pm on May 15." + } + ) +} +``` + +You can also reopen an incident by updating the state from `RESOLVED` to `ACTIVE`. + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "updateIncidentStatus": true + }, + "extensions": {} +} +``` + + + + + +``` +Python SDK support coming soon! +``` + + + \ No newline at end of file diff --git a/docs/api/tutorials/operations.md b/docs/api/tutorials/operations.md new file mode 100644 index 0000000000000..70ede993ec95f --- /dev/null +++ b/docs/api/tutorials/operations.md @@ -0,0 +1,136 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Operations + +## Why Would You Use Operations APIs? + +The Operations APIs allow you to report operational changes that were made to a given Dataset or Table using the 'Operation' concept. +These operations may be viewed on the Dataset Profile (e.g. as last modified time), accessed via the DataHub GraphQL API, or +used to as inputs to Acryl Cloud [Freshness Assertions](/docs/managed-datahub/observe/freshness-assertions.md). + +### Goal Of This Guide + +This guide will show you how to report and query Operations for a Dataset. + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed steps, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +:::note +Before reporting operations for a dataset, you need to ensure the targeted dataset is already present in DataHub. +::: + +## Report Operations + +You can use report dataset operations to DataHub using the following APIs. + + + + +```graphql +mutation reportOperation { + reportOperation( + input: { + urn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + operationType: INSERT, + sourceType: DATA_PROCESS + } + ) +} +``` + +Where supported operation types include + +- `INSERT` +- `UPDATE` +- `DELETE` +- `CREATE` +- `ALTER` +- `DROP` +- `CUSTOM` + +If you want to report an operation that happened at a specific time, you can also optionally provide +the `timestampMillis` field. If not provided, the current server time will be used as the operation time. + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "reportOperation": true + }, + "extensions": {} +} +``` + + + + + +```python +{{ inline /metadata-ingestion/examples/library/dataset_report_operation.py show_path_as_comment }} +``` + + + + +## Read Operations + +You can use read dataset operations to DataHub using the following APIs. + + + + +```graphql +query dataset { + dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)") { + operations( + limit: 10, filter: [], startTimeMillis: , endTimeMillis: + ) { + timestampMillis + operationType + sourceType + } + } +} +``` + +Where startTimeMillis and endTimeMillis are optional. By default, operations are sorted by time descending. + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "dataset": { + "operations": [ + { + "timestampMillis": 1231232332, + "operationType": "INSERT", + "sourceType": "DATA_PROCESS" + } + ] + } + }, + "extensions": {} +} +``` + + + + + +```python +{{ inline /metadata-ingestion/examples/library/dataset_read_operations.py show_path_as_comment }} +``` + + + + +### Expected Outcomes of Reporting Operations + +Reported Operations will appear when displaying the Last Updated time for a Dataset on their DataHub Profile. +They will also be used when selecting the `DataHub Operation` source type under the **Advanced** settings of a Freshness +Assertion. \ No newline at end of file diff --git a/docs/api/tutorials/structured-properties.md b/docs/api/tutorials/structured-properties.md index c32e92e58e8c7..940f4632f1d45 100644 --- a/docs/api/tutorials/structured-properties.md +++ b/docs/api/tutorials/structured-properties.md @@ -16,9 +16,11 @@ Learn more about structured properties in the [Structured Properties Feature Gui This guide will show you how to execute the following actions with structured properties. - Create structured properties - Read structured properties -- Delete structured properties (soft delete) +- Delete structured properties - Add structured properties to a dataset - Patch structured properties (add / remove / update a single property) +- Update structured property with breaking schema changes +- Search & aggregations using structured properties ## Prerequisites @@ -87,9 +89,9 @@ datahub properties upsert -f {properties_yaml} If successful, you should see `Created structured property urn:li:structuredProperty:...` - + -```commandline +```shell curl -X 'POST' -v \ 'http://localhost:8080/openapi/v2/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/propertyDefinition' \ -H 'accept: application/json' \ @@ -120,6 +122,83 @@ curl -X 'POST' -v \ ] }' | jq ``` + + + + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/propertyDefinition' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "qualifiedName": "io.acryl.privacy.retentionTime", + "valueType": "urn:li:dataType:datahub.number", + "description": "Retention Time is used to figure out how long to retain records in a dataset", + "displayName": "Retention Time", + "cardinality": "MULTIPLE", + "entityTypes": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "allowedValues": [ + { + "value": {"double": 30}, + "description": "30 days, usually reserved for datasets that are ephemeral and contain pii" + }, + { + "value": {"double": 60}, + "description": "Use this for datasets that drive monthly reporting but contain pii" + }, + { + "value": {"double": 365}, + "description": "Use this for non-sensitive data that can be retained for longer" + } + ] +}' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "propertyDefinition": { + "value": { + "allowedValues": [ + { + "description": "30 days, usually reserved for datasets that are ephemeral and contain pii", + "value": { + "double": 30 + } + }, + { + "description": "Use this for datasets that drive monthly reporting but contain pii", + "value": { + "double": 60 + } + }, + { + "description": "Use this for non-sensitive data that can be retained for longer", + "value": { + "double": 365 + } + } + ], + "displayName": "Retention Time", + "qualifiedName": "io.acryl.privacy.retentionTime", + "valueType": "urn:li:dataType:datahub.number", + "description": "Retention Time is used to figure out how long to retain records in a dataset", + "entityTypes": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "cardinality": "MULTIPLE" + } + } +} +``` + @@ -137,7 +216,7 @@ datahub properties get --urn {urn} For example, you can run `datahub properties get --urn urn:li:structuredProperty:io.acryl.privacy.retentionTime`. If successful, you should see metadata about your properties returned. -```commandline +```json { "urn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", "qualified_name": "io.acryl.privacy.retentionTime", @@ -167,7 +246,8 @@ If successful, you should see metadata about your properties returned. ``` - + + Example Request: ``` @@ -178,7 +258,7 @@ curl -X 'GET' -v \ Example Response: -```commandline +```json { "value": { "allowedValues": [ @@ -214,6 +294,58 @@ Example Response: } ``` + + + + +Example Request: +``` +curl -X 'GET' -v \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/propertyDefinition' \ + -H 'accept: application/json' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "propertyDefinition": { + "value": { + "allowedValues": [ + { + "description": "30 days, usually reserved for datasets that are ephemeral and contain pii", + "value": { + "double": 30 + } + }, + { + "description": "Use this for datasets that drive monthly reporting but contain pii", + "value": { + "double": 60 + } + }, + { + "description": "Use this for non-sensitive data that can be retained for longer", + "value": { + "double": 365 + } + } + ], + "displayName": "Retention Time", + "qualifiedName": "io.acryl.privacy.retentionTime", + "valueType": "urn:li:dataType:datahub.number", + "description": "Retention Time is used to figure out how long to retain records in a dataset", + "entityTypes": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "cardinality": "MULTIPLE" + } + } +} +``` + @@ -250,12 +382,13 @@ If successful, you should see `Update succeeded for urn:li:dataset:...` - -Following command will set structured properties `retentionTime` as `90` to a dataset `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)`. + + +Following command will set structured properties `retentionTime` as `60.0` to a dataset `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)`. Please note that the structured property and the dataset must exist before executing this command. (You can create sample datasets using the `datahub docker ingest-sample-data`) -```commandline +```shell curl -X 'POST' -v \ 'http://localhost:8080/openapi/v2/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ -H 'accept: application/json' \ @@ -265,13 +398,57 @@ curl -X 'POST' -v \ { "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", "values": [ - {"string": "90"} + {"double": 60.0} + ] + } + ] +}' | jq +``` + + + + + +Following command will set structured properties `retentionTime` as `60.0` to a dataset `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)`. +Please note that the structured property and the dataset must exist before executing this command. (You can create sample datasets using the `datahub docker ingest-sample-data`) + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + {"double": 60.0} ] } ] }' | jq ``` +Example Response: +```json +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "structuredProperties": { + "value": { + "properties": [ + { + "values": [ + { + "double": 60 + } + ], + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime" + } + ] + } + } +} +``` @@ -299,11 +476,11 @@ For this example, we'll extend create a second structured property and apply bot After this your system should include both `io.acryl.privacy.retentionTime` and `io.acryl.privacy.retentionTime02`. - + Let's start by creating the second structured property. -``` +```shell curl -X 'POST' -v \ 'http://localhost:8080/openapi/v2/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime02/propertyDefinition' \ -H 'accept: application/json' \ @@ -331,10 +508,10 @@ curl -X 'POST' -v \ ``` This command will attach one of each of the two properties to our test dataset `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)` -Specically, this will set `io.acryl.privacy.retentionTime` as `90` and `io.acryl.privacy.retentionTime02` as `bar2`. +Specifically, this will set `io.acryl.privacy.retentionTime` as `60.0` and `io.acryl.privacy.retentionTime02` as `bar2`. -``` +```shell curl -X 'POST' -v \ 'http://localhost:8080/openapi/v2/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ -H 'accept: application/json' \ @@ -344,7 +521,99 @@ curl -X 'POST' -v \ { "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", "values": [ - {"string": "90"} + {"double": 60.0} + ] + }, + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02", + "values": [ + {"string": "bar2"} + ] + } + ] +}' | jq +``` + + + + + +Let's start by creating the second structured property. + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime02/propertyDefinition' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "qualifiedName": "io.acryl.privacy.retentionTime02", + "displayName": "Retention Time 02", + "valueType": "urn:li:dataType:datahub.string", + "allowedValues": [ + { + "value": {"string": "foo2"}, + "description": "test foo2 value" + }, + { + "value": {"string": "bar2"}, + "description": "test bar2 value" + } + ], + "cardinality": "SINGLE", + "entityTypes": [ + "urn:li:entityType:datahub.dataset" + ] +}' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02", + "propertyDefinition": { + "value": { + "allowedValues": [ + { + "value": { + "string": "foo2" + }, + "description": "test foo2 value" + }, + { + "value": { + "string": "bar2" + }, + "description": "test bar2 value" + } + ], + "entityTypes": [ + "urn:li:entityType:datahub.dataset" + ], + "qualifiedName": "io.acryl.privacy.retentionTime02", + "displayName": "Retention Time 02", + "cardinality": "SINGLE", + "valueType": "urn:li:dataType:datahub.string" + } + } +} +``` + +This command will attach one of each of the two properties to our test dataset `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)` +Specically, this will set `io.acryl.privacy.retentionTime` as `60.0` and `io.acryl.privacy.retentionTime02` as `bar2`. + + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + {"double": 60.0} ] }, { @@ -357,6 +626,36 @@ curl -X 'POST' -v \ }' | jq ``` +Example Response: + +```json +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "structuredProperties": { + "value": { + "properties": [ + { + "values": [ + { + "double": 60 + } + ], + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime" + }, + { + "values": [ + { + "string": "bar2" + } + ], + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02" + } + ] + } + } +} +``` + @@ -375,9 +674,9 @@ The expected state of our test dataset include 2 structured properties. We'd like to remove the first one (`io.acryl.privacy.retentionTime`) and preserve the second property. (`io.acryl.privacy.retentionTime02`). - + -``` +```shell curl -X 'PATCH' -v \ 'http://localhost:8080/openapi/v2/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ -H 'accept: application/json' \ @@ -398,7 +697,7 @@ curl -X 'PATCH' -v \ ``` The response will show that the expected property has been removed. -``` +```json { "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", "aspects": { @@ -420,6 +719,51 @@ The response will show that the expected property has been removed. } ``` + + + +```shell +curl -X 'PATCH' -v \ + 'http://localhost:8080/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json-patch+json' \ + -d '{ + "patch": [ + { + "op": "remove", + "path": "/properties/urn:li:structuredProperty:io.acryl.privacy.retentionTime" + } + ], + "arrayPrimaryKeys": { + "properties": [ + "propertyUrn" + ] + } + }' | jq +``` +The response will show that the expected property has been removed. + +```json +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "structuredProperties": { + "value": { + "properties": [ + { + "values": [ + { + "string": "bar2" + } + ], + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02" + } + ] + } + } +} +``` + + #### Expected Outcomes @@ -436,9 +780,9 @@ You can see that the first property has been removed and the second property is In this example, we'll add the property back with a different value, preserving the existing property. - + -``` +```shell curl -X 'PATCH' -v \ 'http://localhost:8080/openapi/v2/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ -H 'accept: application/json' \ @@ -452,7 +796,7 @@ curl -X 'PATCH' -v \ "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", "values": [ { - "string": "365" + "double": 365.0 } ] } @@ -466,8 +810,9 @@ curl -X 'PATCH' -v \ }' | jq ``` -Below is the expected response: -``` +Example Response: + +```json { "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", "aspects": { @@ -485,7 +830,7 @@ Below is the expected response: { "values": [ { - "string": "365" + "double": 365.0 } ], "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime" @@ -497,26 +842,91 @@ Below is the expected response: } ``` -The response shows that the property was re-added with the new value bar instead of the previous value foo. +The response shows that the property was re-added with the new value 365.0 instead of the previous value 60.0. - - -#### Expected Outcomes -You can see that the first property has been added back with a new value and the second property is still present. - -

- -

+ +```shell +curl -X 'PATCH' -v \ + 'http://localhost:8080/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json-patch+json' \ + -d '{ + "patch": [ + { + "op": "add", + "path": "/properties/urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "value": { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + { + "double": 365.0 + } + ] + } + } + ], + "arrayPrimaryKeys": { + "properties": [ + "propertyUrn" + ] + } + }' | jq +``` -## Delete Structured Properties - -There are two types of deletion present in DataHub: hard and soft delete. As of the current release only the soft delete is supported for Structured Properties. +Example Response: + +```json +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "structuredProperties": { + "value": { + "properties": [ + { + "values": [ + { + "string": "bar2" + } + ], + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02" + }, + { + "values": [ + { + "double": 365 + } + ], + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime" + } + ] + } + } +} +``` + +The response shows that the property was re-added with the new value 365 instead of the previous value 60. + + + + + +#### Expected Outcomes +You can see that the first property has been added back with a new value and the second property is still present. + +

+ +

+ + +## Delete Structured Properties + +There are two types of deletion present in DataHub: hard and soft delete. :::note SOFT DELETE -A soft deleted Structured Property does not remove any underlying data on the Structured Property entity or the Structured Property's values written to other entities. The soft delete is 100% reversible with zero data loss. When a Structured Property is soft deleted, a few operations are not available. +A soft deleted Structured Property does not remove any underlying data on the Structured Property entity or the Structured Property's values written to other entities. +The soft delete is 100% reversible with zero data loss. When a Structured Property is soft deleted, a few operations are not available. Structured Property Soft Delete Effects: @@ -526,6 +936,19 @@ Structured Property Soft Delete Effects: - Search filters using a soft deleted Structured Property will be denied ::: +:::note HARD DELETE +A hard deleted Structured Property REMOVES all underlying data for the Structured Property entity and the Structured Property's values written to other entities. +The hard delete is NOT reversible. + +Structured Property Hard Delete Effects: + +- Structured Property entity is removed +- Structured Property values are removed via PATCH MCPs on their respective entities +- Rollback is not possible +- Elasticsearch index mappings will continue to contain references to the hard deleted property until reindex +::: + +### Soft Delete @@ -537,11 +960,11 @@ datahub delete --urn {urn} ``` - + The following command will soft delete the test property by writing to the status aspect. -``` +```shell curl -X 'POST' \ 'http://localhost:8080/openapi/v2/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/status?systemMetadata=false' \ -H 'accept: application/json' \ @@ -553,7 +976,7 @@ curl -X 'POST' \ If you want to **remove the soft delete**, you can do so by either hard deleting the status aspect or changing the removed boolean to `false` like below. -``` +```shell curl -X 'POST' \ 'http://localhost:8080/openapi/v2/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/status?systemMetadata=false' \ -H 'accept: application/json' \ @@ -564,4 +987,611 @@ curl -X 'POST' \ ``` + + + +The following command will soft delete the test property by writing to the status aspect. + +```shell +curl -X 'POST' \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/status?systemMetadata=false' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ +"removed": true +}' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "status": { + "value": { + "removed": true + } + } +} +``` + +If you want to **remove the soft delete**, you can do so by either hard deleting the status aspect or changing the removed boolean to `false` like below. + +```shell +curl -X 'POST' \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/status?systemMetadata=false' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ +"removed": false +}' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "status": { + "value": { + "removed": false + } + } +} +``` + + + + + +### Hard Delete + + + + +The following command will hard delete the test property. + +```commandline +datahub delete --urn {urn} --hard +``` + + + + + +The following command will hard delete the test property. + +```shell +curl -v -X 'DELETE' \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime' +``` + +Example Response: + +```text +> DELETE /openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime HTTP/1.1 +> Host: localhost:8080 +> User-Agent: curl/8.4.0 +> Accept: */* +> +< HTTP/1.1 200 OK +< Date: Fri, 14 Jun 2024 17:30:27 GMT +< Content-Length: 0 +< Server: Jetty(11.0.19) +``` + + + +#### Index Mappings Cleanup + +After the asynchronous delete of all Structured Property values have been processed, triggered by the above +hard delete, it is possible to remove the remaining index mappings. Note that if even 1 Structured Property value remains +the mapping will not be removed for a given entity index. + +Run the DataHub system-update job (automatically run with every helm upgrade or install and quickstart) with +the following environment variables enabled. + +This will trigger an ES index which will take time to complete. During the process the entire index is recreated. + +```shell +ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true +ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE=true +``` + +## Update Structured Property With Breaking Schema Changes + +This section will demonstrate how to make backwards incompatible schema changes. Making backwards incompatible +schema changes will remove previously written data. + +Breaking schema changes are implemented by setting a version string within the Structured Property definition. This +version must be in the following format: `yyyyMMddhhmmss`, i.e. `20240614080000` + +:::note IMPORTANT NOTES +Old values will not be retrieve-able after the new Structured Property definition is applied. + +The old values will be subject to deletion asynchronously (future work). +::: + +In the following example, we'll revisit the `retentionTime` structured property and apply a breaking change +by changing the cardinality from `MULTIPLE` to `SINGLE`. Normally this change would be rejected as a +backwards incompatible change since values that were previously written may have multiple values written +which would no longer be valid. + + + + +Edit the previously created definition yaml: Change the cardinality to `SINGLE` and add a `version`. + +```yaml +- id: io.acryl.privacy.retentionTime + # - urn: urn:li:structuredProperty:io.acryl.privacy.retentionTime # optional if id is provided + qualified_name: io.acryl.privacy.retentionTime # required if urn is provided + type: number + cardinality: SINGLE + version: '20240614080000' + display_name: Retention Time + entity_types: + - dataset # or urn:li:entityType:datahub.dataset + - dataFlow + description: "Retention Time is used to figure out how long to retain records in a dataset" + allowed_values: + - value: 30 + description: 30 days, usually reserved for datasets that are ephemeral and contain pii + - value: 90 + description: Use this for datasets that drive monthly reporting but contain pii + - value: 365 + description: Use this for non-sensitive data that can be retained for longer +``` + +Use the CLI to create your properties: +```commandline +datahub properties upsert -f {properties_yaml} +``` + +If successful, you should see `Created structured property urn:li:structuredProperty:...` + + + + +Change the cardinality to `SINGLE` and add a `version`. + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/propertyDefinition' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "qualifiedName": "io.acryl.privacy.retentionTime", + "valueType": "urn:li:dataType:datahub.number", + "description": "Retention Time is used to figure out how long to retain records in a dataset", + "displayName": "Retention Time", + "cardinality": "SINGLE", + "version": "20240614080000", + "entityTypes": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "allowedValues": [ + { + "value": {"double": 30}, + "description": "30 days, usually reserved for datasets that are ephemeral and contain pii" + }, + { + "value": {"double": 60}, + "description": "Use this for datasets that drive monthly reporting but contain pii" + }, + { + "value": {"double": 365}, + "description": "Use this for non-sensitive data that can be retained for longer" + } + ] +}' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "propertyDefinition": { + "value": { + "allowedValues": [ + { + "description": "30 days, usually reserved for datasets that are ephemeral and contain pii", + "value": { + "double": 30 + } + }, + { + "description": "Use this for datasets that drive monthly reporting but contain pii", + "value": { + "double": 60 + } + }, + { + "description": "Use this for non-sensitive data that can be retained for longer", + "value": { + "double": 365 + } + } + ], + "displayName": "Retention Time", + "qualifiedName": "io.acryl.privacy.retentionTime", + "valueType": "urn:li:dataType:datahub.number", + "description": "Retention Time is used to figure out how long to retain records in a dataset", + "entityTypes": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "version": "20240614080000", + "cardinality": "SINGLE" + } + } +} +``` + + + + +## Structured Properties - Search & Aggregation + +Currently Structured Properties can be used to filter search results. This currently excludes fulltext search. + +The following examples re-use the two previously defined Structured Properties. + +`io.acryl.privacy.retentionTime` - An example numeric property. + +`io.acryl.privacy.retentionTime02` - An example string property. + + + + +Range Query: + +Document should be returned based on the previously assigned value of 60. + +```graphql +query { + scrollAcrossEntities( + input: { + types: DATASET, + count: 10, + query: "*", + orFilters: { + and: [ + { + field: "structuredProperties.io.acryl.privacy.retentionTime", + condition: GREATER_THAN, + values: [ + "45.0" + ] + } + ] + } + } + ) { + searchResults { + entity { + urn, + type + } + } + } +} +``` + +Exists Query: + +Document should be returned based on the previously assigned value. + +```graphql +query { + scrollAcrossEntities( + input: { + types: DATASET, + count: 10, + query: "*", + orFilters: { + and: [ + { + field: "structuredProperties.io.acryl.privacy.retentionTime", + condition: EXISTS + } + ] + } + } + ) { + searchResults { + entity { + urn, + type + } + } + } +} +``` + +Equality Query: + +Document should be returned based on the previously assigned value of 'bar2'. + +```graphql +query { + scrollAcrossEntities( + input: { + types: DATASET, + count: 10, + query: "*", + orFilters: { + and: [ + { + field: "structuredProperties.io.acryl.privacy.retentionTime02", + condition: EQUAL + values: [ + "bar2" + ] + } + ] + } + } + ) { + searchResults { + entity { + urn, + type + } + } + } +} +``` + + + + + +Unlike GraphQL which has a parsed input object for filtering, OpenAPI only includes a structured query which +relies on the `query_string` syntax. See the Elasticsearch [documentation](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-query-string-query.html) for detailed syntax. + +In order to use the `query_string` syntax we'll need to know a bit about the Structured Property's definition such +as whether it is versioned or un-unversioned and its type. This information will be added to the `query` url parameter. + +Un-versioned Example: + +Structured Property URN - `urn:li:structuredProperty:io.acryl.privacy.retentionTime` + +Elasticsearch Field Name - `structuredProperties.io_acryl_privacy_retentionTime` + +Versioned: + +Structured Property Version - `20240614080000` + +Structured Property Type - `string` + +Structured Property URN - `urn:li:structuredProperty:io.acryl.privacy.retentionTime02` + +Elasticsearch Field Name - `structuredProperties._versioned.io_acryl_privacy_retentionTime02.20240614080000.string` + +Range Query: + +query - `structuredProperties.io_acryl_privacy_retentionTime:>45` + +```shell +curl -X 'GET' \ + 'http://localhost:9002/openapi/v3/entity/dataset?systemMetadata=false&aspects=datasetKey&aspects=structuredProperties&count=10&sort=urn&sortOrder=ASCENDING&query=structuredProperties.io_acryl_privacy_retentionTime%3A%3E45' \ + -H 'accept: application/json' +``` + +Example Response: + +```json +{ + "entities": [ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "datasetKey": { + "value": { + "name": "SampleHiveDataset", + "platform": "urn:li:dataPlatform:hive", + "origin": "PROD" + } + }, + "structuredProperties": { + "value": { + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + { + "double": 60 + } + ] + }, + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02", + "values": [ + { + "string": "bar2" + } + ] + } + ] + } + } + } + ] +} +``` + +Exists Query: + +query - `_exists_:structuredProperties.io_acryl_privacy_retentionTime` + +```shell +curl -X 'GET' \ + 'http://localhost:9002/openapi/v3/entity/dataset?systemMetadata=false&aspects=datasetKey&aspects=structuredProperties&count=10&sort=urn&sortOrder=ASCENDING&query=_exists_%3AstructuredProperties.io_acryl_privacy_retentionTime' \ + -H 'accept: application/json' +``` + +Example Response: + +```json +{ + "entities": [ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "datasetKey": { + "value": { + "name": "SampleHiveDataset", + "platform": "urn:li:dataPlatform:hive", + "origin": "PROD" + } + }, + "structuredProperties": { + "value": { + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + { + "double": 60 + } + ] + }, + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02", + "values": [ + { + "string": "bar2" + } + ] + } + ] + } + } + } + ] +} +``` + +Equality Query: + +query - `structuredProperties._versioned.io_acryl_privacy_retentionTime02.20240614080000.string` + +```shell +curl -X 'GET' \ + 'http://localhost:9002/openapi/v3/entity/dataset?systemMetadata=false&aspects=datasetKey&aspects=structuredProperties&count=10&sort=urn&sortOrder=ASCENDING&query=structuredProperties._versioned.io_acryl_privacy_retentionTime02.20240614080000.string' \ + -H 'accept: application/json' +``` + +Example Response: + +```json +{ + "entities": [ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "datasetKey": { + "value": { + "name": "SampleHiveDataset", + "platform": "urn:li:dataPlatform:hive", + "origin": "PROD" + } + }, + "structuredProperties": { + "value": { + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + { + "double": 60 + } + ] + }, + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02", + "values": [ + { + "string": "bar2" + } + ] + } + ] + } + } + } + ] +} +``` + + + + +### Structured Property Aggregations + +Structured properties can also be used in GraphQL's aggregation queries using the same naming convention outlined above +for search filter field names. There are currently no aggregation endpoints for OpenAPI. + + + + +Aggregation Query: + +```graphql +query { + aggregateAcrossEntities( + input: { + types: [], + facets: [ + "structuredProperties.io.acryl.privacy.retentionTime02", + "structuredProperties.io.acryl.privacy.retentionTime"], + query: "*", + orFilters: [], + searchFlags: {maxAggValues: 100} + }) { + facets { + field + aggregations { + value + count + } + } + } +} +``` + +Example Response: + +```json +{ + "data": { + "aggregateAcrossEntities": { + "facets": [ + { + "field": "structuredProperties.io.acryl.privacy.retentionTime02", + "aggregations": [ + { + "value": "bar2", + "count": 1 + } + ] + }, + { + "field": "structuredProperties.io.acryl.privacy.retentionTime", + "aggregations": [ + { + "value": "60.0", + "count": 1 + } + ] + } + ] + } + }, + "extensions": {} +} +``` + + + \ No newline at end of file diff --git a/docs/api/tutorials/tags.md b/docs/api/tutorials/tags.md index 24d583dc26dac..a97f8946991f5 100644 --- a/docs/api/tutorials/tags.md +++ b/docs/api/tutorials/tags.md @@ -275,6 +275,8 @@ Expected Response: ### Add Tags to a Column of a dataset +In the example below `subResource` is `fieldPath` in the schema. + diff --git a/docs/assertions/open-assertions-spec.md b/docs/assertions/open-assertions-spec.md new file mode 100644 index 0000000000000..519e917c30587 --- /dev/null +++ b/docs/assertions/open-assertions-spec.md @@ -0,0 +1,486 @@ +# DataHub Open Data Quality Assertions Specification + +DataHub is developing an open-source Data Quality Assertions Specification & Compiler that will allow you to declare data quality checks / expectations / assertions using a simple, universal +YAML-based format, and then compile this into artifacts that can be registered or directly executed by 3rd party Data Quality tools like [Snowflake DMFs](https://docs.snowflake.com/en/user-guide/data-quality-intro), +dbt tests, Great Expectations or Acryl Cloud natively. + +Ultimately, our goal is to provide an framework-agnostic, highly-portable format for defining Data Quality checks, making it seamless to swap out the underlying +assertion engine without service disruption for end consumers of the results of these data quality checks in catalogging tools like DataHub. + +## Integrations + +Currently, the DataHub Open Assertions Specification supports the following integrations: + +- [Snowflake DMF Assertions](snowflake/snowflake_dmfs.md) + +And is looking for contributions to build out support for the following integrations: + +- [Looking for Contributions] dbt tests +- [Looking for Contributions] Great Expectation checks + +Below, we'll look at how to define assertions in YAML, and then provide an usage overview for each support integration. + +## The Specification: Declaring Data Quality Assertions in YAML + +The following assertion types are currently supported by the DataHub YAML Assertion spec: + +- [Freshness](/docs/managed-datahub/observe/freshness-assertions.md) +- [Volume](/docs/managed-datahub/observe/volume-assertions.md) +- [Column](/docs/managed-datahub/observe/column-assertions.md) +- [Custom SQL](/docs/managed-datahub/observe/custom-sql-assertions.md) +- [Schema](/docs/managed-datahub/observe/schema-assertions.md) + +Each assertion type aims to validate a different aspect of structured table (e.g. on a data warehouse or data lake), from +structure to size to column integrity to custom metrics. + +In this section, we'll go over examples of defining each. + +### Freshness Assertions + +Freshness Assertions allow you to verify that your data was updated within the expected timeframe. +Below you'll find examples of defining different types of freshness assertions via YAML. + +#### Validating that Table is Updated Every 6 Hours + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: freshness + lookback_interval: '6 hours' + last_modified_field: updated_at + schedule: + type: interval + interval: '6 hours' # Run every 6 hours +``` + +This assertion checks that the `purchase_events` table in the `test_db.public` schema was updated within the last 6 hours +by issuing a Query to the table which validates determines whether an update was made using the `updated_at` column in the past 6 hours. +To use this check, we must specify the field that contains the last modified timestamp of a given row. + +The `lookback_interval` field is used to specify the "lookback window" for the assertion, whereas the `schedule` field is used to specify how often the assertion should be run. +This allows you to schedule the assertion to run at a different frequency than the lookback window, for example +to detect stale data as soon as it becomes "stale" by inspecting it more frequently. + +#### Supported Source Types + +Currently, the only supported `sourceType` for Freshness Assertions is `LAST_MODIFIED_FIELD`. In the future, +we may support additional source types, such as `HIGH_WATERMARK`, along with data source-specific types such as +`AUDIT_LOG` and `INFORMATION_SCHEMA`. + + +### Volume Assertions + +Volume Assertions allow you to verify that the number of records in your dataset meets your expectations. +Below you'll find examples of defining different types of volume assertions via YAML. + +#### Validating that Tale Row Count is in Expected Range + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: volume + metric: 'row_count' + condition: + type: between + min: 1000 + max: 10000 + # filters: "event_type = 'purchase'" Optionally add filters. + schedule: + type: on_table_change # Run when new data is added to the table. +``` + +This assertion checks that the `purchase_events` table in the `test_db.public` schema has between 1000 and 10000 records. +Using the `condition` field, you can specify the type of comparison to be made, and the `min` and `max` fields to specify the range of values to compare against. +Using the `filters` field, you can optionally specify a SQL WHERE clause to filter the records being counted. +Using the `schedule` field you can specify when the assertion should be run, either on a fixed schedule or when new data is added to the table. +The only metric currently supported is `row_count`. + +#### Validating that Table Row Count is Less Than Value + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: volume + metric: 'row_count' + condition: + type: less_than_or_equal_to + value: 1000 + # filters: "event_type = 'purchase'" Optionally add filters. + schedule: + type: on_table_change # Run when new data is added to the table. +``` + +#### Validating that Table Row Count is Greater Than Value + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: volume + metric: 'row_count' + condition: + type: greater_than_or_equal_to + value: 1000 + # filters: "event_type = 'purchase'" Optionally add filters. + schedule: + type: on_table_change # Run when new data is added to the table. +``` + + +#### Supported Conditions + +The full set of supported volume assertion conditions include: + +- `equal_to` +- `not_equal_to` +- `greater_than` +- `greater_than_or_equal_to` +- `less_than` +- `less_than_or_equal_to` +- `between` + + +### Column Assertions + +Column Assertions allow you to verify that the values in a column meet your expectations. +Below you'll find examples of defining different types of column assertions via YAML. + +The specification currently supports 2 types of Column Assertions: + +- **Field Value**: Asserts that the values in a column meet a specific condition. +- **Field Metric**: Asserts that a specific metric aggregated across the values in a column meet a specific condition. + +We'll go over examples of each below. + +#### Field Values Assertion: Validating that All Column Values are In Expected Range + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: field + field: amount + condition: + type: between + min: 0 + max: 10 + exclude_nulls: True + # filters: "event_type = 'purchase'" Optionally add filters for Column Assertion. + # failure_threshold: + # type: count + # value: 10 + schedule: + type: on_table_change +``` + +This assertion checks that all values for the `amount` column in the `purchase_events` table in the `test_db.public` schema have values between 0 and 10. +Using the `field` field, you can specify the column to be asserted on, and using the `condition` field, you can specify the type of comparison to be made, +and the `min` and `max` fields to specify the range of values to compare against. +Using the `schedule` field you can specify when the assertion should be run, either on a fixed schedule or when new data is added to the table. +Using the `filters` field, you can optionally specify a SQL WHERE clause to filter the records being counted. +Using the `exclude_nulls` field, you can specify whether to exclude NULL values from the assertion, meaning that +NULL will simply be ignored if encountered, as opposed to failing the check. +Using the `failure_threshold`, we can set a threshold for the number of rows that can fail the assertion before the assertion is considered failed. + +#### Field Values Assertion: Validating that All Column Values are In Expected Set + +The validate a VARCHAR / STRING column that should contain one of a set of values: + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: field + field: product_id + condition: + type: in + value: + - 'product_1' + - 'product_2' + - 'product_3' + exclude_nulls: False + # filters: "event_type = 'purchase'" Optionally add filters for Column Assertion. + # failure_threshold: + # type: count + # value: 10 + schedule: + type: on_table_change +``` + +#### Field Values Assertion: Validating that All Column Values are Email Addresses + +The validate a string column contains valid email addresses: + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: field + field: email_address + condition: + type: matches_regex + value: "[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}" + exclude_nulls: False + # filters: "event_type = 'purchase'" Optionally add filters for Column Assertion. + # failure_threshold: + # type: count + # value: 10 + schedule: + type: on_table_change +``` + +#### Field Values Assertion: Supported Conditions + +The full set of supported field value conditions include: + +- `in` +- `not_in` +- `is_null` +- `is_not_null` +- `equal_to` +- `not_equal_to` +- `greater_than` # Numeric Only +- `greater_than_or_equal_to` # Numeric Only +- `less_than` # Numeric Only +- `less_than_or_equal_to` # Numeric Only +- `between` # Numeric Only +- `matches_regex` # String Only +- `not_empty` # String Only +- `length_greater_than` # String Only +- `length_less_than` # String Only +- `length_between` # String Only + + +#### Field Metric Assertion: Validating No Missing Values in Column + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: field + field: col_date + metric: null_count + condition: + type: equal_to + value: 0 + # filters: "event_type = 'purchase'" Optionally add filters for Column Assertion. + schedule: + type: on_table_change +``` + +This assertion ensures that the `col_date` column in the `purchase_events` table in the `test_db.public` schema has no NULL values. + +#### Field Metric Assertion: Validating No Duplicates in Column + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: field + field: id + metric: unique_percentage + condition: + type: equal_to + value: 100 + # filters: "event_type = 'purchase'" Optionally add filters for Column Assertion. + schedule: + type: on_table_change +``` + +This assertion ensures that the `id` column in the `purchase_events` table in the `test_db.public` schema +has no duplicates, by checking that the unique percentage is 100%. + +#### Field Metric Assertion: Validating String Column is Never Empty String + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: field + field: name + metric: empty_percentage + condition: + type: equal_to + value: 0 + # filters: "event_type = 'purchase'" Optionally add filters for Column Assertion. + schedule: + type: on_table_change +``` + +This assertion ensures that the `name` column in the `purchase_events` table in the `test_db.public` schema is never empty, by checking that the empty percentage is 0%. + +#### Field Metric Assertion: Supported Metrics + +The full set of supported field metrics include: + +- `null_count` +- `null_percentage` +- `unique_count` +- `unique_percentage` +- `empty_count` +- `empty_percentage` +- `min` +- `max` +- `mean` +- `median` +- `stddev` +- `negative_count` +- `negative_percentage` +- `zero_count` +- `zero_percentage` + +### Field Metric Assertion: Supported Conditions + +The full set of supported field metric conditions include: + +- `equal_to` +- `not_equal_to` +- `greater_than` +- `greater_than_or_equal_to` +- `less_than` +- `less_than_or_equal_to` +- `between` + +### Custom SQL Assertions + +Custom SQL Assertions allow you to define custom SQL queries to verify your data meets your expectations. +The only condition is that the SQL query must return a single value, which will be compared against the expected value. +Below you'll find examples of defining different types of custom SQL assertions via YAML. + +SQL Assertions are useful for more complex data quality checks that can't be easily expressed using the other assertion types, +and can be used to assert on custom metrics, complex aggregations, cross-table integrity checks (JOINS) or any other SQL-based data quality check. + +#### Validating Foreign Key Integrity + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: sql + statement: | + SELECT COUNT(*) + FROM test_db.public.purchase_events AS pe + LEFT JOIN test_db.public.products AS p + ON pe.product_id = p.id + WHERE p.id IS NULL + condition: + type: equal_to + value: 0 + schedule: + type: interval + interval: '6 hours' # Run every 6 hours +``` + +This assertion checks that the `purchase_events` table in the `test_db.public` schema has no rows where the `product_id` column does not have a corresponding `id` in the `products` table. + +#### Comparing Row Counts Across Multiple Tables + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: sql + statement: | + SELECT COUNT(*) FROM test_db.public.purchase_events + - (SELECT COUNT(*) FROM test_db.public.purchase_events_raw) AS row_count_difference + condition: + type: equal_to + value: 0 + schedule: + type: interval + interval: '6 hours' # Run every 6 hours +``` + +This assertion checks that the number of rows in the `purchase_events` exactly matches the number of rows in an upstream `purchase_events_raw` table +by subtracting the row count of the raw table from the row count of the processed table. + +#### Supported Conditions + +The full set of supported custom SQL assertion conditions include: + +- `equal_to` +- `not_equal_to` +- `greater_than` +- `greater_than_or_equal_to` +- `less_than` +- `less_than_or_equal_to` +- `between` + + +### Schema Assertions (Coming Soon) + +Schema Assertions allow you to define custom SQL queries to verify your data meets your expectations. +Below you'll find examples of defining different types of custom SQL assertions via YAML. + +The specification currently supports 2 types of Schema Assertions: + +- **Exact Match**: Asserts that the schema of a table - column names and their data types - exactly matches an expected schema +- **Contains Match** (Subset): Asserts that the schema of a table - column names and their data types - is a subset of an expected schema + +#### Validating Actual Schema Exactly Equals Expected Schema + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: schema + condition: + type: exact_match + columns: + - name: id + type: INTEGER + - name: product_id + type: STRING + - name: amount + type: DECIMAL + - name: updated_at + type: TIMESTAMP + schedule: + type: interval + interval: '6 hours' # Run every 6 hours +``` + +This assertion checks that the `purchase_events` table in the `test_db.public` schema has the exact schema as specified, with the exact column names and data types. + +#### Validating Actual Schema is Contains all of Expected Schema + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: schema + condition: + type: contains + columns: + - name: id + type: integer + - name: product_id + type: string + - name: amount + type: number + schedule: + type: interval + interval: '6 hours' # Run every 6 hours +``` + +This assertion checks that the `purchase_events` table in the `test_db.public` schema contains all of the columns specified in the expected schema, with the exact column names and data types. +The actual schema can also contain additional columns not specified in the expected schema. + +#### Supported Data Types + +The following high-level data types are currently supported by the Schema Assertion spec: + +- string +- number +- boolean +- date +- timestamp +- struct +- array +- map +- union +- bytes +- enum diff --git a/docs/assertions/snowflake/snowflake_dmfs.md b/docs/assertions/snowflake/snowflake_dmfs.md new file mode 100644 index 0000000000000..e7801a5cbb71b --- /dev/null +++ b/docs/assertions/snowflake/snowflake_dmfs.md @@ -0,0 +1,224 @@ +# Snowflake DMF Assertions [BETA] + +The DataHub Open Assertion Compiler allows you to define your Data Quality assertions in a simple YAML format, and then compile them to be executed by Snowflake Data Metric Functions. +Once compiled, you'll be able to register the compiled DMFs in your Snowflake environment, and extract their results them as part of your normal ingestion process for DataHub. +Results of Snowflake DMF assertions will be reported as normal Assertion Results, viewable on a historical timeline in the context +of the table with which they are associated. + +## Prerequisites + +- You must have a Snowflake Enterprise account, where the DMFs feature is enabled. +- You must have the necessary permissions to provision DMFs in your Snowflake environment (see below) +- You must have the necessary permissions to query the DMF results in your Snowflake environment (see below) +- You must have DataHub instance with Snowflake metadata ingested. If you do not have existing snowflake ingestion, refer [Snowflake Quickstart Guide](https://datahubproject.io/docs/quick-ingestion-guides/snowflake/overview) to get started. +- You must have DataHub CLI installed and run [`datahub init`](https://datahubproject.io/docs/cli/#init). + +### Permissions + +*Permissions required for registering DMFs* + +According to the latest Snowflake docs, here are the permissions the service account performing the +DMF registration and ingestion must have: + +| Privilege | Object | Notes | +|------------------------------|---------------------|---------------------------------------------------------------------------------------------| +| USAGE | Database, schema | Database and schema where snowflake DMFs will be created. This is configured in compile command described below. | +| CREATE FUNCTION | Schema | This privilege enables creating new DMF in schema configured in compile command. | +| EXECUTE DATA METRIC FUNCTION | Account | This privilege enables you to control which roles have access to server-agnostic compute resources to call the system DMF. | +| USAGE | Database, schema | These objects are the database and schema that contain the referenced table in the query. | +| OWNERSHIP | Table | This privilege enables you to associate a DMF with a referenced table. | +| USAGE | DMF | This privilege enables calling the DMF in schema configured in compile command. | + +and the roles that must be granted: + +| Role | Notes | +|--------------------------|-------------------------| +| SNOWFLAKE.DATA_METRIC_USER | To use System DMFs | + +*Permissions required for running DMFs (scheduled DMFs run with table owner's role)* + +Because scheduled DMFs run with the role of the table owner, the table owner must have the following privileges: + +| Privilege | Object | Notes | +|------------------------------|------------------|---------------------------------------------------------------------------------------------| +| USAGE | Database, schema | Database and schema where snowflake DMFs will be created. This is configured in compile command described below. | +| USAGE | DMF | This privilege enables calling the DMF in schema configured in compile power. | +| EXECUTE DATA METRIC FUNCTION | Account | This privilege enables you to control which roles have access to server-agnostic compute resources to call the system DMF. | + +and the roles that must be granted: + +| Role | Notes | +|--------------------------|-------------------------| +| SNOWFLAKE.DATA_METRIC_USER | To use System DMFs | + +*Permissions required for querying DMF results* + +In addition, the service account that will be executing DataHub Ingestion, and querying the DMF results, must have been granted the following system application roles: + +| Role | Notes | +|--------------------------------|-----------------------------| +| DATA_QUALITY_MONITORING_VIEWER | Query the DMF results table | + +To learn more about Snowflake DMFs and the privileges required to provision and query them, see the [Snowflake documentation](https://docs.snowflake.com/en/user-guide/data-quality-intro). + +*Example: Granting Permissions* + +```sql +-- setup permissions to to create DMFs and associate DMFs with table +grant usage on database "" to role "" +grant usage on schema "." to role "" +grant create function on schema "." to role "" +-- grant ownership + rest of permissions to +grant role "" to role "" + +-- setup permissions for to run DMFs on schedule +grant usage on database "" to role "" +grant usage on schema "." to role "" +grant usage on all functions in "." to role "" +grant usage on future functions in "." to role "" +grant database role SNOWFLAKE.DATA_METRIC_USER to role "" +grant execute data metric function on account to role "" + +-- setup permissions for to query DMF results +grant application role SNOWFLAKE.DATA_QUALITY_MONITORING_VIEWER to role "" +``` + +## Supported Assertion Types + +The following assertion types are currently supported by the DataHub Snowflake DMF Assertion Compiler: + +- [Freshness](/docs/managed-datahub/observe/freshness-assertions.md) +- [Volume](/docs/managed-datahub/observe/volume-assertions.md) +- [Column](/docs/managed-datahub/observe/column-assertions.md) +- [Custom SQL](/docs/managed-datahub/observe/custom-sql-assertions.md) + +Note that Schema Assertions are not currently supported. + +## Creating Snowflake DMF Assertions + +The process for declaring and running assertions backend by Snowflake DMFs consists of a few steps, which will be outlined +in the following sections. + + +### Step 1. Define your Data Quality assertions using Assertion YAML files + +See the section **Declaring Assertions in YAML** below for examples of how to define assertions in YAML. + + +### Step 2. Register your assertions with DataHub + +Use the DataHub CLI to register your assertions with DataHub, so they become visible in the DataHub UI: + +```bash +datahub assertions upsert -f examples/library/assertions_configuration.yml +``` + + +### Step 3. Compile the assertions into Snowflake DMFs using the DataHub CLI + +Next, we'll use the `assertions compile` command to generate the SQL code for the Snowflake DMFs, +which can then be registered in Snowflake. + +```bash +datahub assertions compile -f examples/library/assertions_configuration.yml -p snowflake -x DMF_SCHEMA=. +``` + +Two files will be generated as output of running this command: + +- `dmf_definitions.sql`: This file contains the SQL code for the DMFs that will be registered in Snowflake. +- `dmf_associations.sql`: This file contains the SQL code for associating the DMFs with the target tables in Snowflake. + +By default in a folder called `target`. You can use config option `-o ` in `compile` command to write these compile artifacts in another folder. + +Each of these artifacts will be important for the next steps in the process. + +_dmf_definitions.sql_ + +This file stores the SQL code for the DMFs that will be registered in Snowflake, generated +from your YAML assertion definitions during the compile step. + +```sql +-- Example dmf_definitions.sql + +-- Start of Assertion 5c32eef47bd763fece7d21c7cbf6c659 + + CREATE or REPLACE DATA METRIC FUNCTION + test_db.datahub_dmfs.datahub__5c32eef47bd763fece7d21c7cbf6c659 (ARGT TABLE(col_date DATE)) + RETURNS NUMBER + COMMENT = 'Created via DataHub for assertion urn:li:assertion:5c32eef47bd763fece7d21c7cbf6c659 of type volume' + AS + $$ + select case when metric <= 1000 then 1 else 0 end from (select count(*) as metric from TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES ) + $$; + +-- End of Assertion 5c32eef47bd763fece7d21c7cbf6c659 +.... +``` + +_dmf_associations.sql_ + +This file stores the SQL code for associating with the target table, +along with scheduling the generated DMFs to run on at particular times. + +```sql +-- Example dmf_associations.sql + +-- Start of Assertion 5c32eef47bd763fece7d21c7cbf6c659 + + ALTER TABLE TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES SET DATA_METRIC_SCHEDULE = 'TRIGGER_ON_CHANGES'; + ALTER TABLE TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES ADD DATA METRIC FUNCTION test_db.datahub_dmfs.datahub__5c32eef47bd763fece7d21c7cbf6c659 ON (col_date); + +-- End of Assertion 5c32eef47bd763fece7d21c7cbf6c659 +.... +``` + + +### Step 4. Register the compiled DMFs in your Snowflake environment + +Next, you'll need to run the generated SQL from the files output in Step 3 in Snowflake. + +You can achieve this either by running the SQL files directly in the Snowflake UI, or by using the SnowSQL CLI tool: + +```bash +snowsql -f dmf_definitions.sql +snowsql -f dmf_associations.sql +``` + +:::NOTE +Scheduling Data Metric Function on table incurs Serverless Credit Usage in Snowflake. Refer [Billing and Pricing](https://docs.snowflake.com/en/user-guide/data-quality-intro#billing-and-pricing) for more details. +Please ensure you DROP Data Metric Function created via dmf_associations.sql if the assertion is no longer in use. +::: + +### Step 5. Run ingestion to report the results back into DataHub + +Once you've registered the DMFs, they will be automatically executed, either when the target table is updated or on a fixed +schedule. + +To report the results of the generated Data Quality assertions back into DataHub, you'll need to run the DataHub ingestion process with a special configuration +flag: `include_assertion_results: true`: + +```yaml +# Your DataHub Snowflake Recipe +source: + type: snowflake + config: + # ... + include_assertion_results: True + # ... +``` + +During ingestion we will query for the latest DMF results stored in Snowflake, convert them into DataHub Assertion Results, and report the results back into DataHub during your ingestion process +either via CLI or the UI visible as normal assertions. + +`datahub ingest -c snowflake.yml` + +## Caveats + +- Currently, Snowflake supports at most 1000 DMF-table associations at the moment so you can not define more than 1000 assertions for snowflake. +- Currently, Snowflake does not allow JOIN queries or non-deterministic functions in DMF definition so you can not use these in SQL for SQL assertion or in filters section. +- Currently, all DMFs scheduled on a table must follow same exact schedule, so you can not set assertions on same table to run on different schedules. +- Currently, DMFs are only supported for regular tables and not dynamic or external tables. + +## FAQ + +Coming soon! \ No newline at end of file diff --git a/docs/authorization/access-policies-guide.md b/docs/authorization/access-policies-guide.md index a9a54a762cd81..2040d7ff79e99 100644 --- a/docs/authorization/access-policies-guide.md +++ b/docs/authorization/access-policies-guide.md @@ -91,34 +91,8 @@ In the second step, we can simply select the Privileges that this Platform Polic

-**Platform** Privileges most often provide access to perform administrative functions on the Platform. These include: - -| Platform Privileges | Description | -|---------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Manage Policies | Allow actor to create and remove access control policies. Be careful - Actors with this Privilege are effectively super users. | -| Manage Metadata Ingestion | Allow actor to create, remove, and update Metadata Ingestion sources. | -| Manage Secrets | Allow actor to create & remove secrets stored inside DataHub. | -| Manage Users & Groups | Allow actor to create, remove, and update users and groups on DataHub. | -| Manage All Access Tokens | Allow actor to create, remove, and list access tokens for all users on DataHub. | -| Create Domains | Allow the actor to create new Domains | -| Manage Domains | Allow actor to create and remove any Domains. | -| View Analytics | Allow the actor access to the DataHub analytics dashboard. | -| Generate Personal Access Tokens | Allow the actor to generate access tokens for personal use with DataHub APIs. | -| Manage User Credentials | Allow the actor to generate invite links for new native DataHub users, and password reset links for existing native users. | -| Manage Glossaries | Allow the actor to create, edit, move, and delete Glossary Terms and Term Groups | -| Create Tags | Allow the actor to create new Tags | -| Manage Tags | Allow the actor to create and remove any Tags | -| Manage Public Views | Allow the actor to create, edit, and remove any public (shared) Views. | -| Manage Ownership Types | Allow the actor to create, edit, and remove any Ownership Types. | -| Manage Platform Settings | (Acryl DataHub only) Allow the actor to manage global integrations and notification settings | -| Manage Monitors | (Acryl DataHub only) Allow the actor to create, remove, start, or stop any entity assertion monitors | -| Restore Indices API[^1] | Allow the actor to restore indices for a set of entities via API | -| Enable/Disable Writeability API[^1] | Allow the actor to enable or disable GMS writeability for use in data migrations | -| Apply Retention API[^1] | Allow the actor to apply aspect retention via API | -| Explain ElasticSearch Query API[^1] | Allow actor to explain an ElasticSearch query. | - - -[^1]: Only active if REST_API_AUTHORIZATION_ENABLED environment flag is enabled +**Platform** Privileges most often provide access to perform administrative functions on the Platform. +Refer to the [Policies Guide](./policies.md#platform-level-privileges) for a complete list of these privileges. #### Step 3: Choose Policy Actors @@ -194,62 +168,7 @@ scope.

**Metadata** Privileges grant access to change specific *entities* (i.e. data assets) on DataHub. - -The common Metadata Privileges, which span across entity types, include: - -| Common Privileges | Description | -|----------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------| -| View Entity Page | Allow actor to access the entity page for the resource in the UI. If not granted, it will redirect them to an unauthorized page. | -| Edit Tags | Allow actor to add and remove tags to an asset. | -| Edit Glossary Terms | Allow actor to add and remove glossary terms to an asset. | -| Edit Owners | Allow actor to add and remove owners of an entity. | -| Edit Description | Allow actor to edit the description (documentation) of an entity. | -| Edit Links | Allow actor to edit links associated with an entity. | -| Edit Status | Allow actor to edit the status of an entity (soft deleted or not). | -| Edit Domain | Allow actor to edit the Domain of an entity. | -| Edit Deprecation | Allow actor to edit the Deprecation status of an entity. | -| Edit Lineage | Allow actor to edit custom lineage edges for the entity. | -| Edit Data Product | Allow actor to edit the data product that an entity is part of | -| Edit Incidents | Allow actor to raise and resolve incidents associated with an entity. | -| Propose Tags | (Acryl DataHub only) Allow actor to propose new Tags for the entity. | -| Propose Glossary Terms | (Acryl DataHub only) Allow actor to propose new Glossary Terms for the entity. | -| Propose Documentation | (Acryl DataHub only) Allow actor to propose new Documentation for the entity. | -| Manage Tag Proposals | (Acryl DataHub only) Allow actor to accept or reject proposed Tags for the entity. | -| Manage Glossary Terms Proposals | (Acryl DataHub only) Allow actor to accept or reject proposed Glossary Terms for the entity. | -| Manage Documentation Proposals | (Acryl DataHub only) Allow actor to accept or reject proposed Documentation for the entity | -| Edit Entity | Allow actor to edit any information about an entity. Super user privileges. Controls the ability to ingest using API when REST API Authorization is enabled. | -| Get Timeline API[^1] | Allow actor to get the timeline of an entity via API. | -| Get Entity API[^1] | Allow actor to get an entity via API. | -| Get Timeseries Aspect API[^1] | Allow actor to get a timeseries aspect via API. | -| Get Aspect/Entity Count APIs[^1] | Allow actor to get aspect and entity counts via API. | -| Search API | Allow actor to search for entities via API. | -| Produce Platform Event API | Allow actor to ingest a platform event via API. | - -[^1]: Only active if REST_API_AUTHORIZATION_ENABLED is true - -**Specific Metadata Privileges** include - -| Entity | Privilege | Description | -|--------------|------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Dataset | Edit Dataset Column Tags | Allow actor to edit the column (field) tags associated with a dataset schema. | -| Dataset | Edit Dataset Column Glossary Terms | Allow actor to edit the column (field) glossary terms associated with a dataset schema. | -| Dataset | Edit Dataset Column Descriptions | Allow actor to edit the column (field) descriptions associated with a dataset schema. | -| Dataset | Edit Dataset Queries | Allow actor to edit the Highlighted Queries on the Queries tab of the dataset. | -| Dataset | View Dataset Usage | Allow actor to access usage metadata about a dataset both in the UI and in the GraphQL API. This includes example queries, number of queries, etc. Also applies to REST APIs when REST API Authorization is enabled. | -| Dataset | View Dataset Profile | Allow actor to access a dataset's profile both in the UI and in the GraphQL API. This includes snapshot statistics like #rows, #columns, null percentage per field, etc. | -| Dataset | Edit Assertions | Allow actor to change the assertions associated with a dataset. | -| Dataset | Edit Monitors | (Acryl DataHub only) Allow actor to change the assertion monitors associated with a dataset. | -| Tag | Edit Tag Color | Allow actor to change the color of a Tag. | -| Group | Edit Group Members | Allow actor to add and remove members to a group. | -| Group | Edit Contact Information | Allow actor to change email, slack handle associated with the group. | -| Group | Manage Group Subscriptions | (Acryl DataHub only) Allow actor to subscribe the group to entities. | -| Group | Manage Group Notifications | (Acryl DataHub only) Allow actor to change notification settings for the group. | -| User | Edit User Profile | Allow actor to change the user's profile including display name, bio, title, profile image, etc. | -| User + Group | Edit Contact Information | Allow actor to change the contact information such as email & chat handles. | -| Term Group | Manage Direct Glossary Children | Allow actor to change the direct child Term Groups or Terms of the group. | -| Term Group | Manage All Glossary Children | Allow actor to change any direct or indirect child Term Groups or Terms of the group. | - - +These include [**common metadata privileges**](./policies.md#platform-level-privileges) that span across entity types, as well as [**specific entity-level privileges**](./policies.md#specific-entity-level-privileges). #### Step 3: Choose Policy Actors diff --git a/docs/authorization/policies.md b/docs/authorization/policies.md index 759489f291a94..9867ff6ab264d 100644 --- a/docs/authorization/policies.md +++ b/docs/authorization/policies.md @@ -68,75 +68,118 @@ All edits on the UI are covered by a privilege, to make sure we have the ability We currently support the following: -**Platform-level** privileges for DataHub operators to access & manage the administrative functionality of the system. - -| Platform Privileges | Description | -|-------------------------------------|--------------------------------------------------------------------------------------------------------------------------------| -| Manage Policies | Allow actor to create and remove access control policies. Be careful - Actors with this privilege are effectively super users. | -| Manage Metadata Ingestion | Allow actor to create, remove, and update Metadata Ingestion sources. | -| Manage Secrets | Allow actor to create & remove secrets stored inside DataHub. | -| Manage Users & Groups | Allow actor to create, remove, and update users and groups on DataHub. | -| Manage All Access Tokens | Allow actor to create, remove, and list access tokens for all users on DataHub. | -| Create Domains | Allow the actor to create new Domains | -| Manage Domains | Allow actor to create and remove any Domains. | -| View Analytics | Allow the actor access to the DataHub analytics dashboard. | -| Generate Personal Access Tokens | Allow the actor to generate access tokens for personal use with DataHub APIs. | -| Manage User Credentials | Allow the actor to generate invite links for new native DataHub users, and password reset links for existing native users. | -| Manage Glossaries | Allow the actor to create, edit, move, and delete Glossary Terms and Term Groups | -| Create Tags | Allow the actor to create new Tags | -| Manage Tags | Allow the actor to create and remove any Tags | -| Manage Public Views | Allow the actor to create, edit, and remove any public (shared) Views. | -| Restore Indices API[^1] | Allow the actor to restore indices for a set of entities via API | -| Enable/Disable Writeability API[^1] | Allow the actor to enable or disable GMS writeability for use in data migrations | -| Apply Retention API[^1] | Allow the actor to apply aspect retention via API | +##### Platform-level privileges +These privileges are for DataHub operators to access & manage the administrative functionality of the system. + +| Platform Privileges | Description | +|-----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Generate Personal Access Tokens | Allow actor to generate personal access tokens for use with DataHub APIs. | +| Manage Domains | Allow actor to create and remove Asset Domains. | +| Manage Home Page Posts | Allow actor to create and delete home page posts | +| Manage Glossaries | Allow actor to create, edit, and remove Glossary Entities | +| Manage Tags | Allow actor to create and remove Tags. | +| Manage Business Attribute | Allow actor to create, update, delete Business Attribute | +| Manage Documentation Forms | Allow actor to manage forms assigned to assets to assist in documentation efforts. | +| Manage Policies | Allow actor to create and remove access control policies. Be careful - Actors with this privilege are effectively super users. | +| Manage Metadata Ingestion | Allow actor to create, remove, and update Metadata Ingestion sources. | +| Manage Secrets | Allow actor to create & remove Secrets stored inside DataHub. | +| Manage Users & Groups | Allow actor to create, remove, and update users and groups on DataHub. | +| View Analytics | Allow actor to view the DataHub analytics dashboard. | +| Manage All Access Tokens | Allow actor to create, list and revoke access tokens on behalf of users in DataHub. Be careful - Actors with this privilege are effectively super users that can impersonate other users. | +| Manage User Credentials | Allow actor to manage credentials for native DataHub users, including inviting new users and resetting passwords | +| Manage Public Views | Allow actor to create, update, and delete any Public (shared) Views. | +| Manage Ownership Types | Allow actor to create, update and delete Ownership Types. | +| Create Business Attribute | Allow actor to create new Business Attribute. | +| Manage Connections | Allow actor to manage connections to external DataHub platforms. | +| Restore Indices API[^1] | Allow actor to use the Restore Indices API. | +| Get Timeseries index sizes API[^1] | Allow actor to use the get Timeseries indices size API. | +| Truncate timeseries aspect index size API[^1] | Allow actor to use the API to truncate a timeseries index. | +| Get ES task status API[^1] | Allow actor to use the get task status API for an ElasticSearch task. | +| Enable/Disable Writeability API[^1] | Allow actor to enable or disable GMS writeability for data migrations. | +| Apply Retention API[^1] | Allow actor to apply retention using the API. | +| Analytics API access[^1] | Allow actor to use API read access to raw analytics data. | +| Manage Tests[^2] | Allow actor to create and remove Asset Tests. | +| View Metadata Proposals[^2] | Allow actor to view the requests tab for viewing metadata proposals. | +| Create metadata constraints[^2] | Allow actor to create metadata constraints. | +| Manage Platform Settings[^2] | Allow actor to view and change platform-level settings, like integrations & notifications. | +| Manage Monitors[^2] | Allow actor to create, update, and delete any data asset monitors, including Custom SQL monitors. Grant with care. | [^1]: Only active if REST_API_AUTHORIZATION_ENABLED is true - -**Common metadata privileges** to view & modify any entity within DataHub. - -| Common Privileges | Description | -|-------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| View Entity Page | Allow actor to access the entity page for the resource in the UI. If not granted, it will redirect them to an unauthorized page. Additionally if the actor does not have this view privilege, the entity will be removed from search results. | -| Edit Tags | Allow actor to add and remove tags to an asset. | -| Edit Glossary Terms | Allow actor to add and remove glossary terms to an asset. | -| Edit Owners | Allow actor to add and remove owners of an entity. | -| Edit Description | Allow actor to edit the description (documentation) of an entity. | -| Edit Links | Allow actor to edit links associated with an entity. | -| Edit Status | Allow actor to edit the status of an entity (soft deleted or not). | -| Edit Domain | Allow actor to edit the Domain of an entity. | -| Edit Deprecation | Allow actor to edit the Deprecation status of an entity. | -| Edit Assertions | Allow actor to add and remove assertions from an entity. | -| Edit Incidents | Allow actor to raise and resolve incidents for an entity. | -| Edit All | Allow actor to edit any information about an entity. Super user privileges. Controls the ability to ingest using API when REST API Authorization is enabled. | -| Get Timeline API[^1] | Allow actor to get the timeline of an entity via API. | -| Get Entity API[^1] | Allow actor to get an entity via API. | -| Get Timeseries Aspect API[^1] | Allow actor to get a timeseries aspect via API. | -| Get Aspect/Entity Count APIs[^1] | Allow actor to get aspect and entity counts via API. | -| Search API[^1] | Allow actor to search for entities via API. | -| Produce Platform Event API[^1] | Allow actor to ingest a platform event via API. | -| Explain ElasticSearch Query API[^1] | Allow actor to explain an ElasticSearch query. | -| Create Entity | Allow creation of the entity if it doesn't already exist. | -| Entity Exists | Allow checking the existence of the entity without any additional access to the entity's data. | +[^2]: Managed DataHub only + +##### Common metadata privileges +These privileges are to view & modify any entity within DataHub. + +| Common Privileges | Description | +|-------------------------------------|--------------------------------------------------------------------------------------------| +| View Entity Page | Allow actor to view the entity page. | +| Edit Tags | Allow actor to add and remove tags to an asset. | +| Edit Glossary Terms | Allow actor to add and remove glossary terms to an asset. | +| Edit Description | Allow actor to edit the description (documentation) of an entity. | +| Edit Links | Allow actor to edit links associated with an entity. | +| Edit Status | Allow actor to edit the status of an entity (soft deleted or not). | +| Edit Domain | Allow actor to edit the Domain of an entity. | +| Edit Data Product | Allow actor to edit the Data Product of an entity. | +| Edit Deprecation | Allow actor to edit the Deprecation status of an entity. | +| Edit Incidents | Allow actor to create and remove incidents for an entity. | +| Edit Entity | Allow actor to edit any information about an entity. Super user privileges for the entity. | +| Edit Lineage | Allow actor to add and remove lineage edges for this entity. | +| Edit Properties | Allow actor to edit the properties for an entity. | +| Edit Owners | Allow actor to add and remove owners of an entity. | +| Delete | Allow actor to delete this entity. | +| Search API[^1] | Allow actor to access search APIs. | +| Get Aspect/Entity Count APIs[^1] | Allow actor to use the GET Aspect/Entity Count APIs. | +| Get Timeseries Aspect API[^1] | Allow actor to use the GET Timeseries Aspect API. | +| Get Entity + Relationships API[^1] | Allow actor to use the GET Entity and Relationships API. | +| Get Timeline API[^1] | Allow actor to use the GET Timeline API. | +| Explain ElasticSearch Query API[^1] | Allow actor to use the Operations API explain endpoint. | +| Produce Platform Event API[^1] | Allow actor to produce Platform Events using the API. | +| Create Entity | Allow actor to create an entity if it doesn't exist. | +| Entity Exists | Allow actor to determine whether the entity exists. | +| View Entity[^2] | Allow actor to view the entity in search results. | +| Propose Tags[^2] | Allow actor to propose adding a tag to an asset. | +| Propose Glossary Terms[^2] | Allow actor to propose adding a glossary term to an asset. | +| Propose Documentation[^2] | Allow actor to propose updates to an asset's documentation. | +| Manage Tag Proposals[^2] | Allow actor to manage a proposal to add a tag to an asset. | +| Manage Glossary Term Proposals[^2] | Allow actor to manage a proposal to add a glossary term to an asset. | +| Manage Documentation Proposals[^2] | Allow actor to manage a proposal update an asset's documentation | +| Share Entity[^2] | Allow actor to share an entity with another Acryl instance. | [^1]: Only active if REST_API_AUTHORIZATION_ENABLED is true - -**Specific entity-level privileges** that are not generalizable. - -| Entity | Privilege | Description | -|--------------|------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Dataset | Edit Dataset Column Tags | Allow actor to edit the column (field) tags associated with a dataset schema. | -| Dataset | Edit Dataset Column Glossary Terms | Allow actor to edit the column (field) glossary terms associated with a dataset schema. | -| Dataset | Edit Dataset Column Descriptions | Allow actor to edit the column (field) descriptions associated with a dataset schema. | -| Dataset | View Dataset Usage | Allow actor to access usage metadata about a dataset both in the UI and in the GraphQL API. This includes example queries, number of queries, etc. Also applies to REST APIs when REST API Authorization is enabled. | -| Dataset | View Dataset Profile | Allow actor to access a dataset's profile both in the UI and in the GraphQL API. This includes snapshot statistics like #rows, #columns, null percentage per field, etc. | -| Tag | Edit Tag Color | Allow actor to change the color of a Tag. | -| Group | Edit Group Members | Allow actor to add and remove members to a group. | -| User | Edit User Profile | Allow actor to change the user's profile including display name, bio, title, profile image, etc. | -| User + Group | Edit Contact Information | Allow actor to change the contact information such as email & chat handles. | -| GlossaryNode | Manage Direct Glossary Children | Allow the actor to create, edit, and delete the direct children of the selected entities. | -| GlossaryNode | Manage All Glossary Children | Allow the actor to create, edit, and delete everything underneath the selected entities. | - - +[^2]: Managed DataHub only + +##### Specific entity-level privileges +These privileges are not generalizable. + +| Entity | Privilege | Description | +|--------------|-------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Dataset | View Dataset Usage | Allow actor to access dataset usage information (includes usage statistics and queries). | +| Dataset | View Dataset Profile | Allow actor to access dataset profile (snapshot statistics) | +| Dataset | Edit Dataset Column Descriptions | Allow actor to edit the column (field) descriptions associated with a dataset schema. | +| Dataset | Edit Dataset Column Tags | Allow actor to edit the column (field) tags associated with a dataset schema. | +| Dataset | Edit Dataset Column Glossary Terms | Allow actor to edit the column (field) glossary terms associated with a dataset schema. | +| Dataset | Propose Dataset Column Glossary Terms[^2] | Allow actor to propose column (field) glossary terms associated with a dataset schema. | +| Dataset | Propose Dataset Column Tags[^2] | Allow actor to propose new column (field) tags associated with a dataset schema. | +| Dataset | Manage Dataset Column Glossary Terms[^2] | Allow actor to manage column (field) glossary term proposals associated with a dataset schema. | +| Dataset | Propose Dataset Column Descriptions[^2] | Allow actor to propose new descriptions associated with a dataset schema. | +| Dataset | Manage Dataset Column Tag Proposals[^2] | Allow actor to manage column (field) tag proposals associated with a dataset schema. | +| Dataset | Edit Assertions | Allow actor to add and remove assertions from an entity. | +| Dataset | Edit Dataset Queries | Allow actor to edit the Queries for a Dataset. | +| Dataset | Create erModelRelationship | Allow actor to add erModelRelationship on a dataset. | +| Dataset | Edit Monitors[^2] | Allow actor to edit monitors for the entity. | +| Dataset | Edit SQL Assertion Monitors[^2] | Allow actor to edit custom SQL assertion monitors for the entity. Note that this gives read query access to users with through the Custom SQL assertion builder. Grant with care. | +| Dataset | Edit Data Contract[^2] | Allow actor to edit the Data Contract for an entity. | +| Dataset | Manage Data Contract Proposals[^2] | Allow actor to manage a proposal for a Data Contract | +| Tag | Edit Tag Color | Allow actor to change the color of a Tag. | +| Domain | Manage Data Products | Allow actor to create, edit, and delete Data Products within a Domain | +| GlossaryNode | Manage Direct Glossary Children | Allow actor to create and delete the direct children of this entity. | +| GlossaryNode | Manage All Glossary Children | Allow actor to create and delete everything underneath this entity. | +| Group | Edit Group Members | Allow actor to add and remove members to a group. | +| Group | Manage Group Notification Settings[^2] | Allow actor to manage notification settings for a group. | +| Group | Manage Group Subscriptions[^2] | Allow actor to manage subscriptions for a group. | +| Group | Edit Contact Information | Allow actor to change the contact information such as email & chat handles. | +| User | Edit Contact Information | Allow actor to change the contact information such as email & chat handles. | +| User | Edit User Profile | Allow actor to change the user's profile including display name, bio, title, profile image, etc. | #### Resources diff --git a/docs/authorization/roles.md b/docs/authorization/roles.md index 7e2f1797309df..fe41cae2bc3cc 100644 --- a/docs/authorization/roles.md +++ b/docs/authorization/roles.md @@ -79,46 +79,72 @@ These privileges are common to both Self-Hosted DataHub and Managed DataHub. ##### Platform Privileges -| Privilege | Admin | Editor | Reader | -|---------------------------------|--------------------|--------------------|--------| -| Generate Personal Access Tokens | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Manage Domains | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Manage Glossaries | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Manage Tags | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Manage Policies | :heavy_check_mark: | :x: | :x: | -| Manage Ingestion | :heavy_check_mark: | :x: | :x: | -| Manage Secrets | :heavy_check_mark: | :x: | :x: | -| Manage Users and Groups | :heavy_check_mark: | :x: | :x: | -| Manage Access Tokens | :heavy_check_mark: | :x: | :x: | -| Manage User Credentials | :heavy_check_mark: | :x: | :x: | -| Manage Public Views | :heavy_check_mark: | :x: | :x: | -| View Analytics | :heavy_check_mark: | :x: | :x: | +| Privilege | Admin | Editor | Reader | Description | +|-------------------------------------------|--------------------|--------------------|--------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Generate Personal Access Tokens | :heavy_check_mark: | :heavy_check_mark: | :x: | Generate personal access tokens for use with DataHub APIs. | +| Manage Domains | :heavy_check_mark: | :heavy_check_mark: | :x: | Create and remove Asset Domains. | +| Manage Home Page Posts | :heavy_check_mark: | :heavy_check_mark: | :x: | Create and delete home page posts | +| Manage Glossaries | :heavy_check_mark: | :heavy_check_mark: | :x: | Create, edit, and remove Glossary Entities | +| Manage Tags | :heavy_check_mark: | :heavy_check_mark: | :x: | Create and remove Tags. | +| Manage Business Attribute | :heavy_check_mark: | :heavy_check_mark: | :x: | Create, update, delete Business Attribute | +| Manage Documentation Forms | :heavy_check_mark: | :heavy_check_mark: | :x: | Manage forms assigned to assets to assist in documentation efforts. | +| Manage Policies | :heavy_check_mark: | :x: | :x: | Create and remove access control policies. Be careful - Actors with this privilege are effectively super users. | +| Manage Metadata Ingestion | :heavy_check_mark: | :x: | :x: | Create, remove, and update Metadata Ingestion sources. | +| Manage Secrets | :heavy_check_mark: | :x: | :x: | Create & remove Secrets stored inside DataHub. | +| Manage Users & Groups | :heavy_check_mark: | :x: | :x: | Create, remove, and update users and groups on DataHub. | +| View Analytics | :heavy_check_mark: | :x: | :x: | View the DataHub analytics dashboard. | +| Manage All Access Tokens | :heavy_check_mark: | :x: | :x: | Create, list and revoke access tokens on behalf of users in DataHub. Be careful - Actors with this privilege are effectively super users that can impersonate other users. | +| Manage User Credentials | :heavy_check_mark: | :x: | :x: | Manage credentials for native DataHub users, including inviting new users and resetting passwords | +| Manage Public Views | :heavy_check_mark: | :x: | :x: | Create, update, and delete any Public (shared) Views. | +| Manage Ownership Types | :heavy_check_mark: | :x: | :x: | Create, update and delete Ownership Types. | +| Create Business Attribute | :heavy_check_mark: | :x: | :x: | Create new Business Attribute. | +| Manage Connections | :heavy_check_mark: | :x: | :x: | Manage connections to external DataHub platforms. | +| Restore Indices API | :heavy_check_mark: | :x: | :x: | The ability to use the Restore Indices API. | +| Get Timeseries index sizes API | :heavy_check_mark: | :x: | :x: | The ability to use the get Timeseries indices size API. | +| Truncate timeseries aspect index size API | :heavy_check_mark: | :x: | :x: | The ability to use the API to truncate a timeseries index. | +| Get ES task status API | :heavy_check_mark: | :x: | :x: | The ability to use the get task status API for an ElasticSearch task. | +| Enable/Disable Writeability API | :heavy_check_mark: | :x: | :x: | The ability to enable or disable GMS writeability for data migrations. | +| Apply Retention API | :heavy_check_mark: | :x: | :x: | The ability to apply retention using the API. | +| Analytics API access | :heavy_check_mark: | :x: | :x: | API read access to raw analytics data. | ##### Metadata Privileges -| Privilege | Admin | Editor | Reader | -|--------------------------------------|--------------------|--------------------|--------------------| -| View Entity Page | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| View Dataset Usage | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| View Dataset Profile | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Edit Entity | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Edit Entity Tags | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Edit Entity Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Edit Entity Owners | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Edit Entity Docs | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Edit Entity Doc Links | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Edit Entity Status | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Edit Entity Assertions | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Manage Entity Tags | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Manage Entity Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Edit Dataset Column Tags | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Edit Dataset Column Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Edit Dataset Column Descriptions | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Manage Dataset Column Tags | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Manage Dataset Column Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Edit Tag Color | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Edit User Profile | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Edit Contact Info | :heavy_check_mark: | :heavy_check_mark: | :x: | +| Privilege | Admin | Editor | Reader | Description | +|------------------------------------|--------------------|--------------------|--------------------|--------------------------------------------------------------------------------------------------| +| View Entity Page | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to view the entity page. | +| View Dataset Usage | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to access dataset usage information (includes usage statistics and queries). | +| View Dataset Profile | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to access dataset profile (snapshot statistics) | +| Edit Tags | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to add and remove tags to an asset. | +| Edit Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to add and remove glossary terms to an asset. | +| Edit Description | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the description (documentation) of an entity. | +| Edit Links | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit links associated with an entity. | +| Edit Status | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the status of an entity (soft deleted or not). | +| Edit Domain | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the Domain of an entity. | +| Edit Data Product | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the Data Product of an entity. | +| Edit Deprecation | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the Deprecation status of an entity. | +| Edit Assertions | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to add and remove assertions from an entity. | +| Edit Incidents | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to create and remove incidents for an entity. | +| Edit Entity | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit any information about an entity. Super user privileges for the entity. | +| Edit Dataset Column Tags | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the column (field) tags associated with a dataset schema. | +| Edit Dataset Column Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the column (field) glossary terms associated with a dataset schema. | +| Edit Dataset Column Descriptions | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the column (field) descriptions associated with a dataset schema. | +| Edit Tag Color | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to change the color of a Tag. | +| Edit Lineage | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to add and remove lineage edges for this entity. | +| Edit Dataset Queries | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the Queries for a Dataset. | +| Manage Data Products | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to create, edit, and delete Data Products within a Domain | +| Edit Properties | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the properties for an entity. | +| Edit Owners | :heavy_check_mark: | :x: | :x: | The ability to add and remove owners of an entity. | +| Edit Group Members | :heavy_check_mark: | :x: | :x: | The ability to add and remove members to a group. | +| Edit User Profile | :heavy_check_mark: | :x: | :x: | The ability to change the user's profile including display name, bio, title, profile image, etc. | +| Edit Contact Information | :heavy_check_mark: | :x: | :x: | The ability to change the contact information such as email & chat handles. | +| Delete | :heavy_check_mark: | :x: | :x: | The ability to delete this entity. | +| Search API | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to access search APIs. | +| Get Aspect/Entity Count APIs | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to use the GET Aspect/Entity Count APIs. | +| Get Timeseries Aspect API | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to use the GET Timeseries Aspect API. | +| Get Entity + Relationships API | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to use the GET Entity and Relationships API. | +| Get Timeline API | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to use the GET Timeline API. | +| Explain ElasticSearch Query API | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to use the Operations API explain endpoint. | +| Produce Platform Event API | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to produce Platform Events using the API. | #### Managed DataHub @@ -126,22 +152,33 @@ These privileges are only relevant to Managed DataHub. ##### Platform Privileges -| Privilege | Admin | Editor | Reader | -|-------------------------|--------------------|--------------------|--------| -| Create Constraints | :heavy_check_mark: | :heavy_check_mark: | :x: | -| View Metadata Proposals | :heavy_check_mark: | :heavy_check_mark: | :x: | -| Manage Tests | :heavy_check_mark: | :x: | :x: | -| Manage Global Settings | :heavy_check_mark: | :x: | :x: | +| Privilege | Admin | Editor | Reader | Description | +|-----------------------------|--------------------|--------------------|--------|-----------------------------------------------------------------------------------------------------| +| Manage Tests | :heavy_check_mark: | :heavy_check_mark: | :x: | Create and remove Asset Tests. | +| View Metadata Proposals | :heavy_check_mark: | :heavy_check_mark: | :x: | View the requests tab for viewing metadata proposals. | +| Create metadata constraints | :heavy_check_mark: | :heavy_check_mark: | :x: | Create metadata constraints. | +| Manage Platform Settings | :heavy_check_mark: | :x: | :x: | View and change platform-level settings, like integrations & notifications. | +| Manage Monitors | :heavy_check_mark: | :x: | :x: | Create, update, and delete any data asset monitors, including Custom SQL monitors. Grant with care. | ##### Metadata Privileges -| Privilege | Admin | Editor | Reader | -|---------------------------------------|--------------------|--------------------|--------------------| -| Propose Entity Tags | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Propose Entity Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Propose Dataset Column Tags | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Propose Dataset Column Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Edit Entity Operations | :heavy_check_mark: | :heavy_check_mark: | :x: | +| Privilege | Admin | Editor | Reader | Description | +|---------------------------------------|--------------------|--------------------|--------------------|------------------------------------------------------------------------------------------------| +| View Entity | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to view the entity in search results. | +| Propose Tags | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to propose adding a tag to an asset. | +| Propose Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to propose adding a glossary term to an asset. | +| Propose Documentation | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to propose updates to an asset's documentation. | +| Propose Dataset Column Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to propose column (field) glossary terms associated with a dataset schema. | +| Propose Dataset Column Tags | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to propose new column (field) tags associated with a dataset schema. | +| Manage Tag Proposals | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage a proposal to add a tag to an asset. | +| Manage Glossary Term Proposals | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage a proposal to add a glossary term to an asset. | +| Manage Dataset Column Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage column (field) glossary term proposals associated with a dataset schema. | +| Manage Dataset Column Tag Proposals | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage column (field) tag proposals associated with a dataset schema. | +| Manage Documentation Proposals | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage a proposal update an asset's documentation | +| Manage Group Notification Settings | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage notification settings for a group. | +| Manage Group Subscriptions | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage subscriptions for a group. | +| Manage Data Contract Proposals | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage a proposal for a Data Contract | +| Share Entity | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to share an entity with another Acryl instance. | ## Additional Resources diff --git a/docs/cli.md b/docs/cli.md index 411cb2d1ab77f..ca660f209ec42 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -341,6 +341,24 @@ datahub timeline --urn "urn:li:dataset:(urn:li:dataPlatform:mysql,User.UserAccou ## Entity Specific Commands +### dataset (Dataset Entity) + +The `dataset` command allows you to interact with the dataset entity. + +The `get` operation can be used to read in a dataset into a yaml file. +```shell +datahub dataset get --urn "$URN" --to-file "$FILE_NAME" +``` + +The `upsert` operation can be used to create a new user or update an existing one. + +```shell +datahub dataset upsert -f dataset.yaml +``` + +An example of `dataset.yaml` would look like as in [dataset.yaml](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/cli_usage/dataset/dataset.yaml). + + ### user (User Entity) The `user` command allows you to interact with the User entity. @@ -351,7 +369,7 @@ For detailed information, please refer to [Creating Users and Groups with Datahu datahub user upsert -f users.yaml ``` -An example of `users.yaml` would look like the following. You can refer to the [bar.user.dhub.yaml](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/cli_usage/user/bar.user.dhub.yaml) file for the complete code. +An example of `users.yaml` would look like as in [bar.user.dhub.yaml](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/cli_usage/user/bar.user.dhub.yaml) file for the complete code. ```yaml - id: bar@acryl.io @@ -379,7 +397,7 @@ For more information, please refer to [Creating Users and Groups with Datahub CL datahub group upsert -f group.yaml ``` -An example of `group.yaml` would look like the following. You can refer to the [foo.group.dhub.yaml](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/cli_usage/group/foo.group.dhub.yaml) file for the complete code. +An example of `group.yaml` would look like as in [foo.group.dhub.yaml](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/cli_usage/group/foo.group.dhub.yaml) file for the complete code. ```yaml id: foogroup@acryl.io @@ -655,8 +673,8 @@ We use a plugin architecture so that you can install only the dependencies you a Please see our [Integrations page](https://datahubproject.io/integrations) if you want to filter on the features offered by each source. | Plugin Name | Install Command | Provides | -| ---------------------------------------------------------------------------------------------- | ---------------------------------------------------------- | --------------------------------------- | -| [file](./generated/ingestion/sources/file.md) | _included by default_ | File source and sink | +|------------------------------------------------------------------------------------------------| ---------------------------------------------------------- | --------------------------------------- | +| [metadata-file](./generated/ingestion/sources/metadata-file.md) | _included by default_ | File source and sink | | [athena](./generated/ingestion/sources/athena.md) | `pip install 'acryl-datahub[athena]'` | AWS Athena source | | [bigquery](./generated/ingestion/sources/bigquery.md) | `pip install 'acryl-datahub[bigquery]'` | BigQuery source | | [datahub-lineage-file](./generated/ingestion/sources/file-based-lineage.md) | _no additional dependencies_ | Lineage File source | @@ -696,12 +714,12 @@ Please see our [Integrations page](https://datahubproject.io/integrations) if yo ### Sinks -| Plugin Name | Install Command | Provides | -| ----------------------------------------------------------- | -------------------------------------------- | -------------------------- | -| [file](../metadata-ingestion/sink_docs/file.md) | _included by default_ | File source and sink | -| [console](../metadata-ingestion/sink_docs/console.md) | _included by default_ | Console sink | -| [datahub-rest](../metadata-ingestion/sink_docs/datahub.md) | `pip install 'acryl-datahub[datahub-rest]'` | DataHub sink over REST API | -| [datahub-kafka](../metadata-ingestion/sink_docs/datahub.md) | `pip install 'acryl-datahub[datahub-kafka]'` | DataHub sink over Kafka | +| Plugin Name | Install Command | Provides | +|-------------------------------------------------------------------| -------------------------------------------- | -------------------------- | +| [metadata-file](../metadata-ingestion/sink_docs/metadata-file.md) | _included by default_ | File source and sink | +| [console](../metadata-ingestion/sink_docs/console.md) | _included by default_ | Console sink | +| [datahub-rest](../metadata-ingestion/sink_docs/datahub.md) | `pip install 'acryl-datahub[datahub-rest]'` | DataHub sink over REST API | +| [datahub-kafka](../metadata-ingestion/sink_docs/datahub.md) | `pip install 'acryl-datahub[datahub-kafka]'` | DataHub sink over Kafka | These plugins can be mixed and matched as desired. For example: diff --git a/docs/components.md b/docs/components.md index 9ece0056e6862..bc17d98d9e1d4 100644 --- a/docs/components.md +++ b/docs/components.md @@ -38,7 +38,7 @@ either Kafka or using the Metadata Store Rest APIs directly. DataHub supports an a host of capabilities including schema extraction, table & column profiling, usage information extraction, and more. Getting started with the Ingestion Framework is as simple: just define a YAML file and execute the `datahub ingest` command. -Learn more by heading over the the [Metadata Ingestion](https://datahubproject.io/docs/metadata-ingestion/) guide. +Learn more by heading over the [Metadata Ingestion](https://datahubproject.io/docs/metadata-ingestion/) guide. ## GraphQL API diff --git a/docs/deploy/environment-vars.md b/docs/deploy/environment-vars.md index e2354e398ecb9..21ed738e878f8 100644 --- a/docs/deploy/environment-vars.md +++ b/docs/deploy/environment-vars.md @@ -45,23 +45,25 @@ DataHub works. ## Search -| Variable | Default | Unit/Type | Components | Description | -|-----------------------------------------------------|------------------------|-----------|-----------------------------------------------------------------|--------------------------------------------------------------------------| -| `INDEX_PREFIX` | `` | string | [`GMS`, `MAE Consumer`, `Elasticsearch Setup`, `System Update`] | Prefix Elasticsearch indices with the given string. | -| `ELASTICSEARCH_NUM_SHARDS_PER_INDEX` | 1 | integer | [`System Update`] | Default number of shards per Elasticsearch index. | -| `ELASTICSEARCH_NUM_REPLICAS_PER_INDEX` | 1 | integer | [`System Update`] | Default number of replica per Elasticsearch index. | -| `ELASTICSEARCH_BUILD_INDICES_RETENTION_VALUE` | 60 | integer | [`System Update`] | Number of units for the retention of Elasticsearch clone/backup indices. | -| `ELASTICSEARCH_BUILD_INDICES_RETENTION_UNIT` | DAYS | string | [`System Update`] | Unit for the retention of Elasticsearch clone/backup indices. | -| `ELASTICSEARCH_QUERY_EXACT_MATCH_EXCLUSIVE` | `false` | boolean | [`GMS`] | Only return exact matches when using quotes. | -| `ELASTICSEARCH_QUERY_EXACT_MATCH_WITH_PREFIX` | `true` | boolean | [`GMS`] | Include prefix match in exact match results. | -| `ELASTICSEARCH_QUERY_EXACT_MATCH_FACTOR` | 10.0 | float | [`GMS`] | Multiply by this number on true exact match. | -| `ELASTICSEARCH_QUERY_EXACT_MATCH_PREFIX_FACTOR` | 1.6 | float | [`GMS`] | Multiply by this number when prefix match. | -| `ELASTICSEARCH_QUERY_EXACT_MATCH_CASE_FACTOR` | 0.7 | float | [`GMS`] | Multiply by this number when case insensitive match. | -| `ELASTICSEARCH_QUERY_EXACT_MATCH_ENABLE_STRUCTURED` | `true` | boolean | [`GMS`] | When using structured query, also include exact matches. | -| `ELASTICSEARCH_QUERY_PARTIAL_URN_FACTOR` | 0.5 | float | [`GMS`] | Multiply by this number when partial token match on URN) | -| `ELASTICSEARCH_QUERY_PARTIAL_FACTOR` | 0.4 | float | [`GMS`] | Multiply by this number when partial token match on non-URN field. | -| `ELASTICSEARCH_QUERY_CUSTOM_CONFIG_ENABLED` | `false` | boolean | [`GMS`] | Enable search query and ranking customization configuration. | -| `ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE` | `search_config.yml` | string | [`GMS`] | The location of the search customization configuration. | +| Variable | Default | Unit/Type | Components | Description | +|-----------------------------------------------------|---------------------|-----------|-----------------------------------------------------------------|--------------------------------------------------------------------------| +| `INDEX_PREFIX` | `` | string | [`GMS`, `MAE Consumer`, `Elasticsearch Setup`, `System Update`] | Prefix Elasticsearch indices with the given string. | +| `ELASTICSEARCH_NUM_SHARDS_PER_INDEX` | 1 | integer | [`System Update`] | Default number of shards per Elasticsearch index. | +| `ELASTICSEARCH_NUM_REPLICAS_PER_INDEX` | 1 | integer | [`System Update`] | Default number of replica per Elasticsearch index. | +| `ELASTICSEARCH_BUILD_INDICES_RETENTION_VALUE` | 60 | integer | [`System Update`] | Number of units for the retention of Elasticsearch clone/backup indices. | +| `ELASTICSEARCH_BUILD_INDICES_RETENTION_UNIT` | DAYS | string | [`System Update`] | Unit for the retention of Elasticsearch clone/backup indices. | +| `ELASTICSEARCH_QUERY_EXACT_MATCH_EXCLUSIVE` | `false` | boolean | [`GMS`] | Only return exact matches when using quotes. | +| `ELASTICSEARCH_QUERY_EXACT_MATCH_WITH_PREFIX` | `true` | boolean | [`GMS`] | Include prefix match in exact match results. | +| `ELASTICSEARCH_QUERY_EXACT_MATCH_FACTOR` | 10.0 | float | [`GMS`] | Multiply by this number on true exact match. | +| `ELASTICSEARCH_QUERY_EXACT_MATCH_PREFIX_FACTOR` | 1.6 | float | [`GMS`] | Multiply by this number when prefix match. | +| `ELASTICSEARCH_QUERY_EXACT_MATCH_CASE_FACTOR` | 0.7 | float | [`GMS`] | Multiply by this number when case insensitive match. | +| `ELASTICSEARCH_QUERY_EXACT_MATCH_ENABLE_STRUCTURED` | `true` | boolean | [`GMS`] | When using structured query, also include exact matches. | +| `ELASTICSEARCH_QUERY_PARTIAL_URN_FACTOR` | 0.5 | float | [`GMS`] | Multiply by this number when partial token match on URN) | +| `ELASTICSEARCH_QUERY_PARTIAL_FACTOR` | 0.4 | float | [`GMS`] | Multiply by this number when partial token match on non-URN field. | +| `ELASTICSEARCH_QUERY_CUSTOM_CONFIG_ENABLED` | `true` | boolean | [`GMS`] | Enable search query and ranking customization configuration. | +| `ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE` | `search_config.yml` | string | [`GMS`] | The location of the search customization configuration. | +| `ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX` | `false` | boolean | [`System Update`] | Enable reindexing on Elasticsearch schema changes. | +| `ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE` | `false` | boolean | [`System Update`] | Enable reindexing to remove hard deleted structured properties. | ## Kafka diff --git a/docs/glossary/business-glossary.md b/docs/glossary/business-glossary.md index 9c9daabcb94c7..f77dc8d4f7a9e 100644 --- a/docs/glossary/business-glossary.md +++ b/docs/glossary/business-glossary.md @@ -136,6 +136,10 @@ Glossary Terms and Term Groups abide by metadata policies like other entities. H - **Manage Direct Glossary Children**: If a user has this privilege on a Glossary Term Group, they will be able to create, edit, and delete Terms and Term Groups directly underneath the Term Group they have this privilege on. - **Manage All Glossary Children**: If a user has this privilege on a Glossary Term Group, they will be able to create, edit, and delete any Term or Term Group anywhere underneath the Term Group they have this privilege on. This applies to the children of a child Term Group as well (and so on). +## Shift left and bring your glossary into Git + +You can use this [Github Action](https://github.com/acryldata/business-glossary-sync-action) and bring your Business Glossary into your git repositories. This can be the starting point to manage glossary in git. + ## Managing Glossary with Git In many cases, it may be preferable to manage the Business Glossary in a version-control system like git. This can make diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index c29c20e7d48a3..ef9990ca3804e 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -20,11 +20,51 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ### Breaking Changes +- Protobuf CLI will no longer create binary encoded protoc custom properties. Flag added `-protocProp` in case this + behavior is required. +- #10868 - OpenAPI V3 - Creation of aspects will need to be wrapped within a `value` key and the API is now symmetric with respect to input and outputs. + +Example Global Tags Aspect: + +Previous: +```json +{ + "tags": [ + { + "tag": "string", + "context": "string" + } + ] +} +``` + +New (optional fields `systemMetadata` and `headers`): + +```json +{ + "value": { + "tags": [ + { + "tag": "string", + "context": "string" + } + ] + }, + "systemMetadata": {}, + "headers": {} +} +``` + ### Potential Downtime ### Deprecations +- OpenAPI v1: OpenAPI v1 is collectively defined as all endpoints which are not prefixed with `/v2` or `/v3`. The v1 endpoints + will be deprecated in no less than 6 months. Endpoints will be replaced with equivalents in the `/v2` or `/v3` APIs. + No loss of functionality expected unless explicitly mentioned in Breaking Changes. + ### Other Notable Changes +- #10498 - Tableau ingestion can now be configured to ingest multiple sites at once and add the sites as containers. The feature is currently only available for Tableau Server. ## 0.13.3 @@ -484,7 +524,7 @@ Helm with `--atomic`: In general, it is recommended to not use the `--atomic` se ### Breaking Changes -- The `should_overwrite` flag in `csv-enricher` has been replaced with `write_semantics` to match the format used for other sources. See the [documentation](https://datahubproject.io/docs/generated/ingestion/sources/csv/) for more details +- The `should_overwrite` flag in `csv-enricher` has been replaced with `write_semantics` to match the format used for other sources. See the [documentation](https://datahubproject.io/docs/generated/ingestion/sources/csv-enricher/) for more details - Closing an authorization hole in creating tags adding a Platform Privilege called `Create Tags` for creating tags. This is assigned to `datahub` root user, along with default All Users policy. Notice: You may need to add this privilege (or `Manage Tags`) to existing users that need the ability to create tags on the platform. - #5329 Below profiling config parameters are now supported in `BigQuery`: diff --git a/docs/lineage/airflow.md b/docs/lineage/airflow.md index f0952309c328a..62715ed506ffe 100644 --- a/docs/lineage/airflow.md +++ b/docs/lineage/airflow.md @@ -66,11 +66,12 @@ enabled = True # default ``` | Name | Default value | Description | -| -------------------------- | -------------------- | ---------------------------------------------------------------------------------------- | +|----------------------------|----------------------|------------------------------------------------------------------------------------------| | enabled | true | If the plugin should be enabled. | | conn_id | datahub_rest_default | The name of the datahub rest connection. | -| cluster | prod | name of the airflow cluster | +| cluster | prod | name of the airflow cluster, this is equivalent to the `env` of the instance | | capture_ownership_info | true | Extract DAG ownership. | +| capture_ownership_as_group | false | When extracting DAG ownership, treat DAG owner as a group rather than a user | | capture_tags_info | true | Extract DAG tags. | | capture_executions | true | Extract task runs and success/failure statuses. This will show up in DataHub "Runs" tab. | | materialize_iolets | true | Create or un-soft-delete all entities referenced in lineage. | @@ -130,18 +131,19 @@ conn_id = datahub_rest_default # or datahub_kafka_default # etc. ``` -| Name | Default value | Description | -| ---------------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| enabled | true | If the plugin should be enabled. | -| conn_id | datahub_rest_default | The name of the datahub connection you set in step 1. | -| cluster | prod | name of the airflow cluster | -| capture_ownership_info | true | If true, the owners field of the DAG will be capture as a DataHub corpuser. | -| capture_tags_info | true | If true, the tags field of the DAG will be captured as DataHub tags. | -| capture_executions | true | If true, we'll capture task runs in DataHub in addition to DAG definitions. | -| materialize_iolets | true | Create or un-soft-delete all entities referenced in lineage. | -| datajob_url_link | taskinstance | If taskinstance, the datajob url will be taskinstance link on airflow. It can also be grid. | -| | -| graceful_exceptions | true | If set to true, most runtime errors in the lineage backend will be suppressed and will not cause the overall task to fail. Note that configuration issues will still throw exceptions. | +| Name | Default value | Description | +|----------------------------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| enabled | true | If the plugin should be enabled. | +| conn_id | datahub_rest_default | The name of the datahub connection you set in step 1. | +| cluster | prod | name of the airflow cluster | +| capture_ownership_info | true | If true, the owners field of the DAG will be capture as a DataHub corpuser. | +| capture_ownership_as_group | false | When extracting DAG ownership, treat DAG owner as a group rather than a user. | +| capture_tags_info | true | If true, the tags field of the DAG will be captured as DataHub tags. | +| capture_executions | true | If true, we'll capture task runs in DataHub in addition to DAG definitions. | +| materialize_iolets | true | Create or un-soft-delete all entities referenced in lineage. | +| datajob_url_link | taskinstance | If taskinstance, the datajob url will be taskinstance link on airflow. It can also be grid. | +| | +| graceful_exceptions | true | If set to true, most runtime errors in the lineage backend will be suppressed and will not cause the overall task to fail. Note that configuration issues will still throw exceptions. | #### Validate that the plugin is working @@ -233,6 +235,37 @@ You can also create a custom extractor to extract lineage from any operator. Thi See this [example PR](https://github.com/datahub-project/datahub/pull/10452) which adds a custom extractor for the `BigQueryInsertJobOperator` operator. +## Cleanup obsolete pipelines and tasks from Datahub + +There might be a case where the DAGs are removed from the Airflow but the corresponding pipelines and tasks are still there in the Datahub, let's call such pipelines ans tasks, `obsolete pipelines and tasks` + +Following are the steps to cleanup them from the datahub: +- create a DAG named `Datahub_Cleanup`, i.e. + +```python +from datetime import datetime + +from airflow import DAG +from airflow.operators.bash import BashOperator + +from datahub_airflow_plugin.entities import Dataset, Urn + +with DAG( + "Datahub_Cleanup", + start_date=datetime(2024, 1, 1), + schedule_interval=None, + catchup=False, +) as dag: + task = BashOperator( + task_id="cleanup_obsolete_data", + dag=dag, + bash_command="echo 'cleaning up the obsolete data from datahub'", + ) + +``` +- ingest this DAG, and it will remove all the obsolete pipelines and tasks from the Datahub based on the `cluster` value set in the `airflow.cfg` + + ## Emit Lineage Directly If you can't use the plugin or annotate inlets/outlets, you can also emit lineage using the `DatahubEmitterOperator`. diff --git a/docs/managed-datahub/managed-datahub-overview.md b/docs/managed-datahub/managed-datahub-overview.md index 867b03501e0e0..087238097dd9f 100644 --- a/docs/managed-datahub/managed-datahub-overview.md +++ b/docs/managed-datahub/managed-datahub-overview.md @@ -51,10 +51,17 @@ know. | Feature | DataHub | Acryl DataHub | | ---------------------------------------------- | ------- | ------------- | | Surface data quality results | ✅ | ✅ | -| Build and enforce continuous data SLAs | ❌ | ✅ | -| Continuous monitoring of dataset health | ❌ | ✅ | -| Data observability alerts and notifications | ❌ | ✅ | -| Data Incident management | ❌ | ✅ | +| Create data contracts | ✅ | ✅ | +| Raise and Resolve Data Incidents | ✅ | ✅ | +| Monitor Freshness SLAs | ❌ | ✅ | +| Monitor Table Schemas | ❌ | ✅ | +| Monitor Table Volume | ❌ | ✅ | +| Validate Table Columns | ❌ | ✅ | +| Receive Notifications via Email & Slack | ❌ | ✅ | +| Manage Data Incidents via Slack | ❌ | ✅ | +| View Data Health Dashboard | ❌ | ✅ | +| Evaluate data quality checks on-demand (API + UI) | ❌ | ✅ | +| Evaluate data quality checks in your VPC | ❌ | ✅ | ## Enterprise Grade diff --git a/docs/managed-datahub/metadata-ingestion-with-acryl/ingestion.md b/docs/managed-datahub/metadata-ingestion-with-acryl/ingestion.md deleted file mode 100644 index 0444d15b3627c..0000000000000 --- a/docs/managed-datahub/metadata-ingestion-with-acryl/ingestion.md +++ /dev/null @@ -1,116 +0,0 @@ -# Ingestion - -Acryl Metadata Ingestion functions similarly to that in open source DataHub. Sources are configured via the[ UI Ingestion](docs/ui-ingestion.md) or via a [Recipe](metadata-ingestion/README.md#recipes), ingestion recipes can be scheduled using your system of choice, and metadata can be pushed from anywhere. -This document will describe the steps required to ingest metadata from your data sources. - -## Batch Ingestion - -Batch ingestion involves extracting metadata from a source system in bulk. Typically, this happens on a predefined schedule using the [Metadata Ingestion ](metadata-ingestion/README.md#install-from-pypi)framework. -The metadata that is extracted includes point-in-time instances of dataset, chart, dashboard, pipeline, user, group, usage, and task metadata. - -### Step 1: Install DataHub CLI - -Regardless of how you ingest metadata, you'll need your account subdomain and API key handy. - -#### **Install from Gemfury Private Repository** - -**Installing from command line with pip** - -Determine the version you would like to install and obtain a read access token by requesting a one-time-secret from the Acryl team then run the following command: - -`python3 -m pip install acryl-datahub== --index-url https://:@pypi.fury.io/acryl-data/` - -#### Install from PyPI for OSS Release - -Run the following commands in your terminal: - -``` -python3 -m pip install --upgrade pip wheel setuptools -python3 -m pip install --upgrade acryl-datahub -python3 -m datahub version -``` - -_Note: Requires Python 3.6+_ - -Your command line should return the proper version of DataHub upon executing these commands successfully. - -### Step 2: Install Connector Plugins - -Our CLI follows a plugin architecture. You must install connectors for different data sources individually. For a list of all supported data sources, see [the open source docs](metadata-ingestion/README.md#installing-plugins). -Once you've found the connectors you care about, simply install them using `pip install`. -For example, to install the `mysql` connector, you can run - -```python -pip install --upgrade acryl-datahub[mysql] -``` - -### Step 3: Write a Recipe - -[Recipes](metadata-ingestion/README.md#recipes) are yaml configuration files that serve as input to the Metadata Ingestion framework. Each recipe file define a single source to read from and a single destination to push the metadata. -The two most important pieces of the file are the _source_ and _sink_ configuration blocks. -The _source_ configuration block defines where to extract metadata from. This can be an OLTP database system, a data warehouse, or something as simple as a file. Each source has custom configuration depending on what is required to access metadata from the source. To see configurations required for each supported source, refer to the [Sources](metadata-ingestion/README.md#sources) documentation. -The _sink_ configuration block defines where to push metadata into. Each sink type requires specific configurations, the details of which are detailed in the [Sinks](metadata-ingestion/README.md#sinks) documentation. -In Acryl DataHub deployments, you _must_ use a sink of type `datahub-rest`, which simply means that metadata will be pushed to the REST endpoints exposed by your DataHub instance. The required configurations for this sink are - -1. **server**: the location of the REST API exposed by your instance of DataHub -2. **token**: a unique API key used to authenticate requests to your instance's REST API - -The token can be retrieved by logging in as admin. You can go to Settings page and generate a Personal Access Token with your desired expiration date. - -

- -

- - - -

- -

- - -To configure your instance of DataHub as the destination for ingestion, set the "server" field of your recipe to point to your Acryl instance's domain suffixed by the path `/gms`, as shown below. -A complete example of a DataHub recipe file, which reads from MySQL and writes into a DataHub instance: - -```yaml -# example-recipe.yml - -# MySQL source configuration -source: - type: mysql - config: - username: root - password: password - host_port: localhost:3306 - -# Recipe sink configuration. -sink: - type: "datahub-rest" - config: - server: "https://.acryl.io/gms" - token: -``` - -:::info -Your API key is a signed JSON Web Token that is valid for 6 months from the date of issuance. Please keep this key secure & avoid sharing it. -::: - -If your key is compromised for any reason, please reach out to the Acryl team at support@acryl.io.::: - -### Step 4: Running Ingestion - -The final step requires invoking the DataHub CLI to ingest metadata based on your recipe configuration file. -To do so, simply run `datahub ingest` with a pointer to your YAML recipe file: - -``` -datahub ingest -c ./example-recipe.yml -``` - -### Step 5: Scheduling Ingestion - -Ingestion can either be run in an ad-hoc manner by a system administrator or scheduled for repeated executions. Most commonly, ingestion will be run on a daily cadence. -To schedule your ingestion job, we recommend using a job schedule like [Apache Airflow](https://airflow.apache.org/). In cases of simpler deployments, a CRON job scheduled on an always-up machine can also work. -Note that each source system will require a separate recipe file. This allows you to schedule ingestion from different sources independently or together. - -_Looking for information on real-time ingestion? Click_ [_here_](docs/lineage/airflow.md)_._ - -_Note: Real-time ingestion setup is not recommended for an initial POC as it generally takes longer to configure and is prone to inevitable system errors._ diff --git a/docs/managed-datahub/release-notes/v_0_3_3.md b/docs/managed-datahub/release-notes/v_0_3_3.md new file mode 100644 index 0000000000000..6d341a45a183f --- /dev/null +++ b/docs/managed-datahub/release-notes/v_0_3_3.md @@ -0,0 +1,23 @@ +# v0.3.3 +--- + +Release Availability Date +--- +25-June-2024 + +Recommended CLI/SDK +--- +- `v0.13.3` with release notes at https://github.com/acryldata/datahub/releases/tag/v0.13.3 + +If you are using an older CLI/SDK version then please upgrade it. This applies for all CLI/SDK usages, if you are using it through your terminal, github actions, airflow, in python SDK somewhere, Java SKD etc. This is a strong recommendation to upgrade as we keep on pushing fixes in the CLI and it helps us support you better. + +## Release Changelog +--- + +- Ability to run assertions "on-demand" from the Acryl UI (Reach out to Acryl to enable this feature) +- Richer Incident Slack Messages - Resolve and reopen incidents from Slack. Reflect the latest status of incidents in the same slack message. Richer details about incident when raised via Assertion +- Assertion-Level Subscriptions - Subscribe to be notified when a specific assertion passes, fails, or errors out. +- Misc fixes with 2.0 UI (new loading indicators, fixed text overflows, consistent entity health badges) +- Structured Property Schema Change & Delete +- Since `v0.3.2` these changes from OSS DataHub https://github.com/datahub-project/datahub/compare/6ed21bd1bc70a3ceb7dddb43ea7db4ca56874547...92e9a5823bc14e81f0f21c55a68c493c3bbe87b9 have been pulled in. + diff --git a/docs/managed-datahub/subscription-and-notification.md b/docs/managed-datahub/subscription-and-notification.md index b30a03de16511..81648d4298ec1 100644 --- a/docs/managed-datahub/subscription-and-notification.md +++ b/docs/managed-datahub/subscription-and-notification.md @@ -17,6 +17,8 @@ As a user, you can subscribe to and receive notifications about changes such as ## Prerequisites Once you have [configured Slack within your DataHub instance](saas-slack-setup.md), you will be able to subscribe to any Entity in DataHub and begin recieving notifications via DM. +To begin receiving personal notifications, go to Settings > "My Notifications". From here, toggle on Slack Notifications and input your Slack Member ID. + If you want to create and manage group-level Subscriptions for your team, you will need [the following privileges](../../docs/authorization/roles.md#role-privileges): - Manage Group Notification Settings @@ -93,6 +95,39 @@ You can view and manage the group’s subscriptions on the group’s page on Dat

+### Subscribing to Assertions +You can always subscribe to _all assertion status changes_ on a table using the steps outlined in the earlier sections. However, in some cases you may want to only be notified about specific assertions on a table. For instance, a table may contain several subsets of information, segmented by a category column - so there may be several different checks for each category. As a consumer, you may only care about the freshness check that runs on one specific category of this larger table. + +You can subscribe to individual assertions by clicking the bell button on the assertion itself - either in the list view: +

+ 1 +

+ +Or on the assertion's profile page: +

+ 2 +

+ + +Note: if you are subscribed to all assertions at the dataset level, then you will not be able to **Unsubscribe** from an individual assertion. +

+ 3 +

+ +You must first remove your dataset-level subscription: +

+ 4 + 5 +

+ + +Then select individual assertions you'd like to subscribe to: +

+ 7 +

+ + + ## FAQ
diff --git a/docs/managed-datahub/welcome-acryl.md b/docs/managed-datahub/welcome-acryl.md index d13e94f0fa200..e292d6bb0ea11 100644 --- a/docs/managed-datahub/welcome-acryl.md +++ b/docs/managed-datahub/welcome-acryl.md @@ -49,7 +49,7 @@ Acryl DataHub employs a push-based metadata ingestion model. In practice, this m This approach comes with another benefit: security. By managing your own instance of the agent, you can keep the secrets and credentials within your walled garden. Skip uploading secrets & keys into a third-party cloud tool. -To push metadata into DataHub, Acryl provide's an ingestion framework written in Python. Typically, push jobs are run on a schedule at an interval of your choosing. For our step-by-step guide on ingestion, click [here](docs/managed-datahub/metadata-ingestion-with-acryl/ingestion.md). +To push metadata into DataHub, Acryl provide's an ingestion framework written in Python. Typically, push jobs are run on a schedule at an interval of your choosing. For our step-by-step guide on ingestion, click [here](../../metadata-ingestion/cli-ingestion.md). ### Discovering Metadata diff --git a/docs/posts.md b/docs/posts.md index c44125bbd0017..107ea9020031b 100644 --- a/docs/posts.md +++ b/docs/posts.md @@ -13,9 +13,33 @@ DataHub allows users to make Posts that can be displayed on the app. Currently, Anyone can view Posts on the home page. To create Posts, a user must either have the **Create Global Announcements** Privilege, or possess the **Admin** DataHub Role. -## Using Posts +## Creating Posts -To create a post, users must use the [createPost](../graphql/mutations.md#createPost) GraphQL mutation. There is currently no way to create posts using the UI, though this will come in the future. +### Create Posts Using the UI +To create a post, first navigate to the Settings tab in the top-right menu of DataHub. +Once you're on the Settings page, click 'Home Page Posts'. +To create a new Post, click '+ New Post'. +

+ Creating a new post +

+DataHub currently supports two types of Post content. Posts can either be of type **Text** or **Link**. Click on "Post Type" to switch between these types. +

+ Selecting text post type +

+

+ Selecting link post type +

+If you choose the text type, enter the title and description as prompted; if you choose the link type, enter the title and the URL of the link and the address of the image as prompted. + +Click 'Create' to complete. +

+ Viewing posts +

+ +### Create Posts Using the GraphQL + +To create a post via API, you can call the [createPost](../graphql/mutations.md#createPost) GraphQL mutation. +To create a post via API, you can call the [createPost](../graphql/mutations.md#createPost) GraphQL mutation. There is only one type of Post that can be currently made, and that is a **Home Page Announcement**. This may be extended in the future to other surfaces. diff --git a/docs/quick-ingestion-guides/powerbi/setup.md b/docs/quick-ingestion-guides/powerbi/setup.md index 0aed3559fd3e5..0ff52d1351f76 100644 --- a/docs/quick-ingestion-guides/powerbi/setup.md +++ b/docs/quick-ingestion-guides/powerbi/setup.md @@ -70,7 +70,7 @@ In order to configure ingestion from PowerBI, you'll first have to ensure you ha - `Enhance admin APIs responses with detailed metadata` - `Enhance admin APIs responses with DAX and mashup expressions` - f. **Add Security Group to Workspace:** Navigate to `Workspaces` window and open workspace which you want to ingest as shown in below screenshot and click on `Access` and add `powerbi-connector-app-security-group` as member + f. **Add Security Group to Workspace:** Navigate to `Workspaces` window and open workspace which you want to ingest as shown in below screenshot and click on `Access` and add `powerbi-connector-app-security-group` as member. For most cases `Viewer` role is enough, but for profiling the `Contributor` role is required.

workspace-window-underlined diff --git a/docs/troubleshooting/quickstart.md b/docs/troubleshooting/quickstart.md index 9da5aa443069e..cafc1e30c5052 100644 --- a/docs/troubleshooting/quickstart.md +++ b/docs/troubleshooting/quickstart.md @@ -246,7 +246,7 @@ ALTER TABLE metadata_aspect_v2 CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_ ## I've modified the default user.props file to include a custom username and password, but I don't see the new user(s) inside the Users & Groups tab. Why not? Currently, `user.props` is a file used by the JAAS PropertyFileLoginModule solely for the purpose of **Authentication**. The file is not used as an source from which to -ingest additional metadata about the user. For that, you'll need to ingest some custom information about your new user using the Rest.li APIs or the [File-based ingestion source](../generated/ingestion/sources/file.md). +ingest additional metadata about the user. For that, you'll need to ingest some custom information about your new user using the Rest.li APIs or the [Metadata File ingestion source](../generated/ingestion/sources/metadata-file.md). For an example of a file that ingests user information, check out [single_mce.json](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/mce_files/single_mce.json), which ingests a single user object into DataHub. Notice that the "urn" field provided will need to align with the custom username you've provided in user.props file. For example, if your user.props file contains: diff --git a/docs/what-is-datahub/customer-stories.md b/docs/what-is-datahub/customer-stories.md new file mode 100644 index 0000000000000..2745a8aca0d2f --- /dev/null +++ b/docs/what-is-datahub/customer-stories.md @@ -0,0 +1,17 @@ +import CustomerCardSection from '@site/src/pages/docs/_components/CustomerCardSection'; + +# Customer Stories + +Meet the DataHub users who have shared their stories with us. + +:::note Share Your DataHub Journey +We're excited to hear about your experience with DataHub. Share your story with us! + +Share Your Story + +::: + + diff --git a/entity-registry/build.gradle b/entity-registry/build.gradle index 484a1f3271dbb..2dedea1f16d99 100644 --- a/entity-registry/build.gradle +++ b/entity-registry/build.gradle @@ -1,6 +1,7 @@ plugins { id 'pegasus' id 'java-library' + id 'java-test-fixtures' } dependencies { @@ -45,5 +46,8 @@ dependencies { exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind' } + testFixturesImplementation externalDependency.mockito + testFixturesCompileOnly externalDependency.lombok + testFixturesAnnotationProcessor externalDependency.lombok } compileTestJava.dependsOn tasks.getByPath(':entity-registry:custom-test-model:modelDeploy') diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/AspectRetriever.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/AspectRetriever.java index ad47562662fd6..e34df7db48118 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/AspectRetriever.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/AspectRetriever.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.aspect; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; import com.linkedin.entity.Aspect; @@ -31,6 +32,23 @@ default Aspect getLatestAspectObject(@Nonnull final Urn urn, @Nonnull final Stri @Nonnull Map> getLatestAspectObjects(Set urns, Set aspectNames); + @Nullable + default SystemAspect getLatestSystemAspect( + @Nonnull final Urn urn, @Nonnull final String aspectName) { + return getLatestSystemAspects(ImmutableMap.of(urn, ImmutableSet.of(aspectName))) + .getOrDefault(urn, Collections.emptyMap()) + .get(aspectName); + } + + /** + * Returns for each URN, the map of aspectName to Aspect + * + * @param urnAspectNames urns and aspect names to fetch + * @return urn to aspect name and values + */ + @Nonnull + Map> getLatestSystemAspects(Map> urnAspectNames); + @Nonnull default Map entityExists(Set urns) { Set keyAspectNames = diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/EnvelopedSystemAspect.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/EnvelopedSystemAspect.java new file mode 100644 index 0000000000000..b4b8e095a38ac --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/EnvelopedSystemAspect.java @@ -0,0 +1,62 @@ +package com.linkedin.metadata.aspect; + +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.mxe.SystemMetadata; +import java.sql.Timestamp; +import java.time.Instant; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.Getter; + +/** Delegate to restli class */ +public class EnvelopedSystemAspect implements SystemAspect { + + public static SystemAspect of( + @Nonnull Urn urn, @Nonnull EnvelopedAspect envelopedAspect, @Nonnull EntitySpec entitySpec) { + return new EnvelopedSystemAspect(urn, envelopedAspect, entitySpec); + } + + @Getter @Nonnull private final Urn urn; + @Nonnull private final EnvelopedAspect envelopedAspect; + @Getter @Nonnull private final EntitySpec entitySpec; + @Getter @Nonnull private final AspectSpec aspectSpec; + + public EnvelopedSystemAspect( + @Nonnull Urn urn, @Nonnull EnvelopedAspect envelopedAspect, @Nonnull EntitySpec entitySpec) { + this.urn = urn; + this.envelopedAspect = envelopedAspect; + this.entitySpec = entitySpec; + this.aspectSpec = this.entitySpec.getAspectSpec(envelopedAspect.getName()); + } + + @Nullable + @Override + public RecordTemplate getRecordTemplate() { + return envelopedAspect.getValue(); + } + + @Nullable + @Override + public SystemMetadata getSystemMetadata() { + return envelopedAspect.getSystemMetadata(); + } + + @Override + public long getVersion() { + return envelopedAspect.getVersion(); + } + + @Override + public Timestamp getCreatedOn() { + return Timestamp.from(Instant.ofEpochMilli(envelopedAspect.getCreated().getTime())); + } + + @Override + public String getCreatedBy() { + return envelopedAspect.getCreated().getActor().toString(); + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/RetrieverContext.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/RetrieverContext.java index df1b9c6a6259c..629d2c0aad52d 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/RetrieverContext.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/RetrieverContext.java @@ -1,7 +1,11 @@ package com.linkedin.metadata.aspect; +import com.linkedin.metadata.entity.SearchRetriever; + public interface RetrieverContext { GraphRetriever getGraphRetriever(); AspectRetriever getAspectRetriever(); + + SearchRetriever getSearchRetriever(); } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/SystemAspect.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/SystemAspect.java index e83414c8c23a8..4c9bf3d4fdbc7 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/SystemAspect.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/SystemAspect.java @@ -2,7 +2,9 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.mxe.SystemMetadata; import java.sql.Timestamp; +import java.util.Optional; import javax.annotation.Nonnull; /** @@ -22,4 +24,16 @@ default AuditStamp getAuditStamp() { .setActor(UrnUtils.getUrn(getCreatedBy())) .setTime(getCreatedOn().getTime()); } + + /** + * If aspect version exists in system metadata, return it + * + * @return version of the aspect + */ + default Optional getSystemMetadataVersion() { + return Optional.ofNullable(getSystemMetadata()) + .filter(SystemMetadata::hasVersion) + .map(SystemMetadata::getVersion) + .map(Long::parseLong); + } } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java index 031625da0477c..a302632e1936f 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java @@ -129,6 +129,16 @@ static Stream applyMCPSideEffects( .flatMap(mcpSideEffect -> mcpSideEffect.apply(items, retrieverContext)); } + default Stream applyPostMCPSideEffects(Collection items) { + return applyPostMCPSideEffects(items, getRetrieverContext()); + } + + static Stream applyPostMCPSideEffects( + Collection items, @Nonnull RetrieverContext retrieverContext) { + return retrieverContext.getAspectRetriever().getEntityRegistry().getAllMCPSideEffects().stream() + .flatMap(mcpSideEffect -> mcpSideEffect.postApply(items, retrieverContext)); + } + default Stream applyMCLSideEffects(Collection items) { return applyMCLSideEffects(items, getRetrieverContext()); } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/MCPItem.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/MCPItem.java index 96b2752516e60..afe11f56cce50 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/MCPItem.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/MCPItem.java @@ -5,18 +5,42 @@ import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.mxe.MetadataChangeProposal; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; import java.util.Set; +import javax.annotation.Nonnull; import javax.annotation.Nullable; /** Represents a proposal to write to the primary data store which may be represented by an MCP */ public interface MCPItem extends BatchItem { Set CHANGE_TYPES = - ImmutableSet.of(ChangeType.UPSERT, ChangeType.CREATE, ChangeType.CREATE_ENTITY); + ImmutableSet.of( + ChangeType.UPSERT, ChangeType.UPDATE, ChangeType.CREATE, ChangeType.CREATE_ENTITY); @Nullable MetadataChangeProposal getMetadataChangeProposal(); + @Nonnull + default Map getHeaders() { + if (getMetadataChangeProposal() != null && getMetadataChangeProposal().getHeaders() != null) { + return getMetadataChangeProposal().getHeaders(); + } + return Collections.emptyMap(); + } + + default boolean hasHeader(@Nonnull String headerName) { + return getHeaders().keySet().stream().anyMatch(hdr -> hdr.equalsIgnoreCase(headerName)); + } + + default Optional getHeader(@Nonnull String headerName) { + return getHeaders().entrySet().stream() + .filter(entry -> entry.getKey().equalsIgnoreCase(headerName)) + .map(Map.Entry::getValue) + .findAny(); + } + /** * Validates that a change type is valid for the given aspect * diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/FormInfoPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/FormInfoPatchBuilder.java new file mode 100644 index 0000000000000..13d9c07b04aee --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/FormInfoPatchBuilder.java @@ -0,0 +1,154 @@ +package com.linkedin.metadata.aspect.patch.builder; + +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.*; + +import com.datahub.util.RecordUtils; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.linkedin.form.FormPrompt; +import com.linkedin.form.FormType; +import com.linkedin.metadata.aspect.patch.PatchOperationType; +import java.util.List; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.commons.lang3.tuple.ImmutableTriple; + +public class FormInfoPatchBuilder extends AbstractMultiFieldPatchBuilder { + + public static final String PATH_DELIM = "/"; + public static final String NAME_FIELD = "name"; + public static final String DESCRIPTION_FIELD = "description"; + public static final String TYPE_FIELD = "type"; + public static final String PROMPTS_FIELD = "prompts"; + public static final String ACTORS_FIELD = "actors"; + public static final String OWNERS_FIELD = "owners"; + public static final String USERS_FIELD = "users"; + public static final String GROUPS_FIELD = "groups"; + + public FormInfoPatchBuilder setName(@Nonnull String name) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), PATH_DELIM + NAME_FIELD, instance.textNode(name))); + return this; + } + + public FormInfoPatchBuilder setDescription(@Nullable String description) { + if (description == null) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.REMOVE.getValue(), PATH_DELIM + DESCRIPTION_FIELD, null)); + } else { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + DESCRIPTION_FIELD, + instance.textNode(description))); + } + return this; + } + + public FormInfoPatchBuilder setType(@Nonnull FormType formType) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + TYPE_FIELD, + instance.textNode(formType.toString()))); + return this; + } + + public FormInfoPatchBuilder addPrompt(@Nonnull FormPrompt prompt) { + try { + ObjectNode promptNode = + (ObjectNode) new ObjectMapper().readTree(RecordUtils.toJsonString(prompt)); + pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + PROMPTS_FIELD + PATH_DELIM + prompt.getId(), + promptNode)); + return this; + } catch (JsonProcessingException e) { + throw new IllegalArgumentException( + "Failed to add prompt, failed to parse provided aspect json.", e); + } + } + + public FormInfoPatchBuilder addPrompts(@Nonnull List prompts) { + try { + prompts.forEach(this::addPrompt); + return this; + } catch (Exception e) { + throw new IllegalArgumentException("Failed to add prompts.", e); + } + } + + public FormInfoPatchBuilder removePrompt(@Nonnull String promptId) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.REMOVE.getValue(), + PATH_DELIM + PROMPTS_FIELD + PATH_DELIM + promptId, + null)); + return this; + } + + public FormInfoPatchBuilder removePrompts(@Nonnull List promptIds) { + promptIds.forEach(this::removePrompt); + return this; + } + + public FormInfoPatchBuilder setOwnershipForm(boolean isOwnershipForm) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + ACTORS_FIELD + PATH_DELIM + OWNERS_FIELD, + instance.booleanNode(isOwnershipForm))); + return this; + } + + public FormInfoPatchBuilder addAssignedUser(@Nonnull String userUrn) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + ACTORS_FIELD + PATH_DELIM + USERS_FIELD + PATH_DELIM + userUrn, + instance.textNode(userUrn))); + return this; + } + + public FormInfoPatchBuilder removeAssignedUser(@Nonnull String userUrn) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.REMOVE.getValue(), + PATH_DELIM + ACTORS_FIELD + PATH_DELIM + USERS_FIELD + PATH_DELIM + userUrn, + instance.textNode(userUrn))); + return this; + } + + public FormInfoPatchBuilder addAssignedGroup(@Nonnull String groupUrn) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + ACTORS_FIELD + PATH_DELIM + GROUPS_FIELD + PATH_DELIM + groupUrn, + instance.textNode(groupUrn))); + return this; + } + + public FormInfoPatchBuilder removeAssignedGroup(@Nonnull String groupUrn) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.REMOVE.getValue(), + PATH_DELIM + ACTORS_FIELD + PATH_DELIM + GROUPS_FIELD + PATH_DELIM + groupUrn, + instance.textNode(groupUrn))); + return this; + } + + @Override + protected String getAspectName() { + return FORM_INFO_ASPECT_NAME; + } + + @Override + protected String getEntityType() { + return FORM_ENTITY_NAME; + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertiesPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertiesPatchBuilder.java index fab81e0af5bf5..b568df5054aae 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertiesPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertiesPatchBuilder.java @@ -3,12 +3,11 @@ import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.ValueNode; import com.linkedin.common.urn.Urn; import com.linkedin.metadata.aspect.patch.PatchOperationType; import java.util.List; -import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.apache.commons.lang3.tuple.ImmutableTriple; @@ -17,8 +16,8 @@ public class StructuredPropertiesPatchBuilder extends AbstractMultiFieldPatchBuilder { private static final String BASE_PATH = "/properties"; - private static final String URN_KEY = "urn"; - private static final String CONTEXT_KEY = "context"; + private static final String URN_KEY = "propertyUrn"; + private static final String VALUES_KEY = "values"; /** * Remove a property from a structured properties aspect. If the property doesn't exist, this is a @@ -34,63 +33,77 @@ public StructuredPropertiesPatchBuilder removeProperty(Urn propertyUrn) { return this; } - public StructuredPropertiesPatchBuilder setProperty( - @Nonnull Urn propertyUrn, @Nullable List propertyValues) { - propertyValues.stream() - .map( - propertyValue -> - propertyValue instanceof Integer - ? this.setProperty(propertyUrn, (Integer) propertyValue) - : this.setProperty(propertyUrn, String.valueOf(propertyValue))) - .collect(Collectors.toList()); - return this; - } - - public StructuredPropertiesPatchBuilder setProperty( + public StructuredPropertiesPatchBuilder setNumberProperty( @Nonnull Urn propertyUrn, @Nullable Integer propertyValue) { - ValueNode propertyValueNode = instance.numberNode((Integer) propertyValue); - ObjectNode value = instance.objectNode(); - value.put(URN_KEY, propertyUrn.toString()); + ObjectNode newProperty = instance.objectNode(); + newProperty.put(URN_KEY, propertyUrn.toString()); + + ArrayNode valuesNode = instance.arrayNode(); + ObjectNode propertyValueNode = instance.objectNode(); + propertyValueNode.set("double", instance.numberNode(propertyValue)); + valuesNode.add(propertyValueNode); + newProperty.set(VALUES_KEY, valuesNode); + pathValues.add( ImmutableTriple.of( - PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, propertyValueNode)); + PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, newProperty)); return this; } - public StructuredPropertiesPatchBuilder setProperty( - @Nonnull Urn propertyUrn, @Nullable String propertyValue) { - ValueNode propertyValueNode = instance.textNode(String.valueOf(propertyValue)); - ObjectNode value = instance.objectNode(); - value.put(URN_KEY, propertyUrn.toString()); + public StructuredPropertiesPatchBuilder setNumberProperty( + @Nonnull Urn propertyUrn, @Nonnull List propertyValues) { + ObjectNode newProperty = instance.objectNode(); + newProperty.put(URN_KEY, propertyUrn.toString()); + + ArrayNode valuesNode = instance.arrayNode(); + propertyValues.forEach( + propertyValue -> { + ObjectNode propertyValueNode = instance.objectNode(); + propertyValueNode.set("double", instance.numberNode(propertyValue)); + valuesNode.add(propertyValueNode); + }); + newProperty.set(VALUES_KEY, valuesNode); + pathValues.add( ImmutableTriple.of( - PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, propertyValueNode)); + PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, newProperty)); return this; } - public StructuredPropertiesPatchBuilder addProperty( - @Nonnull Urn propertyUrn, @Nullable Integer propertyValue) { - ValueNode propertyValueNode = instance.numberNode((Integer) propertyValue); - ObjectNode value = instance.objectNode(); - value.put(URN_KEY, propertyUrn.toString()); + public StructuredPropertiesPatchBuilder setStringProperty( + @Nonnull Urn propertyUrn, @Nullable String propertyValue) { + ObjectNode newProperty = instance.objectNode(); + newProperty.put(URN_KEY, propertyUrn.toString()); + + ArrayNode valuesNode = instance.arrayNode(); + ObjectNode propertyValueNode = instance.objectNode(); + propertyValueNode.set("string", instance.textNode(propertyValue)); + valuesNode.add(propertyValueNode); + newProperty.set(VALUES_KEY, valuesNode); + pathValues.add( ImmutableTriple.of( - PatchOperationType.ADD.getValue(), - BASE_PATH + "/" + propertyUrn + "/" + String.valueOf(propertyValue), - propertyValueNode)); + PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, newProperty)); return this; } - public StructuredPropertiesPatchBuilder addProperty( - @Nonnull Urn propertyUrn, @Nullable String propertyValue) { - ValueNode propertyValueNode = instance.textNode(String.valueOf(propertyValue)); - ObjectNode value = instance.objectNode(); - value.put(URN_KEY, propertyUrn.toString()); + public StructuredPropertiesPatchBuilder setStringProperty( + @Nonnull Urn propertyUrn, @Nonnull List propertyValues) { + ObjectNode newProperty = instance.objectNode(); + newProperty.put(URN_KEY, propertyUrn.toString()); + + ArrayNode valuesNode = instance.arrayNode(); + propertyValues.forEach( + propertyValue -> { + ObjectNode propertyValueNode = instance.objectNode(); + propertyValueNode.set("string", instance.textNode(propertyValue)); + valuesNode.add(propertyValueNode); + }); + newProperty.set(VALUES_KEY, valuesNode); + pathValues.add( ImmutableTriple.of( - PatchOperationType.ADD.getValue(), - BASE_PATH + "/" + propertyUrn + "/" + String.valueOf(propertyValue), - propertyValueNode)); + PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, newProperty)); return this; } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertyDefinitionPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertyDefinitionPatchBuilder.java new file mode 100644 index 0000000000000..0811e2f52d003 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertyDefinitionPatchBuilder.java @@ -0,0 +1,146 @@ +package com.linkedin.metadata.aspect.patch.builder; + +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.*; + +import com.datahub.util.RecordUtils; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.linkedin.data.template.StringArrayMap; +import com.linkedin.metadata.aspect.patch.PatchOperationType; +import com.linkedin.structured.PropertyCardinality; +import com.linkedin.structured.PropertyValue; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.commons.lang3.tuple.ImmutableTriple; + +public class StructuredPropertyDefinitionPatchBuilder + extends AbstractMultiFieldPatchBuilder { + + public static final String PATH_DELIM = "/"; + public static final String QUALIFIED_NAME_FIELD = "qualifiedName"; + public static final String DISPLAY_NAME_FIELD = "displayName"; + public static final String VALUE_TYPE_FIELD = "valueType"; + public static final String TYPE_QUALIFIER_FIELD = "typeQualifier"; + public static final String ALLOWED_VALUES_FIELD = "allowedValues"; + public static final String CARDINALITY_FIELD = "cardinality"; + public static final String ENTITY_TYPES_FIELD = "entityTypes"; + public static final String DESCRIPTION_FIELD = "description"; + public static final String IMMUTABLE_FIELD = "immutable"; + + // can only be used when creating a new structured property + public StructuredPropertyDefinitionPatchBuilder setQualifiedName(@Nonnull String name) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + QUALIFIED_NAME_FIELD, + instance.textNode(name))); + return this; + } + + public StructuredPropertyDefinitionPatchBuilder setDisplayName(@Nonnull String displayName) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + DISPLAY_NAME_FIELD, + instance.textNode(displayName))); + return this; + } + + public StructuredPropertyDefinitionPatchBuilder setValueType(@Nonnull String valueTypeUrn) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + VALUE_TYPE_FIELD, + instance.textNode(valueTypeUrn))); + return this; + } + + // can only be used when creating a new structured property + public StructuredPropertyDefinitionPatchBuilder setTypeQualifier( + @Nonnull StringArrayMap typeQualifier) { + ObjectNode value = instance.objectNode(); + typeQualifier.forEach( + (key, values) -> { + ArrayNode valuesNode = instance.arrayNode(); + values.forEach(valuesNode::add); + value.set(key, valuesNode); + }); + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), PATH_DELIM + TYPE_QUALIFIER_FIELD, value)); + return this; + } + + public StructuredPropertyDefinitionPatchBuilder addAllowedValue( + @Nonnull PropertyValue propertyValue) { + try { + ObjectNode valueNode = + (ObjectNode) new ObjectMapper().readTree(RecordUtils.toJsonString(propertyValue)); + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + ALLOWED_VALUES_FIELD + PATH_DELIM + propertyValue.getValue(), + valueNode)); + return this; + } catch (JsonProcessingException e) { + throw new IllegalArgumentException( + "Failed to add allowed value, failed to parse provided aspect json.", e); + } + } + + public StructuredPropertyDefinitionPatchBuilder setCardinality( + @Nonnull PropertyCardinality cardinality) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + CARDINALITY_FIELD, + instance.textNode(cardinality.toString()))); + return this; + } + + public StructuredPropertyDefinitionPatchBuilder addEntityType(@Nonnull String entityTypeUrn) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + ENTITY_TYPES_FIELD + PATH_DELIM + entityTypeUrn, + instance.textNode(entityTypeUrn))); + return this; + } + + public StructuredPropertyDefinitionPatchBuilder setDescription(@Nullable String description) { + if (description == null) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.REMOVE.getValue(), PATH_DELIM + DESCRIPTION_FIELD, null)); + } else { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + DESCRIPTION_FIELD, + instance.textNode(description))); + } + return this; + } + + public StructuredPropertyDefinitionPatchBuilder setImmutable(boolean immutable) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + IMMUTABLE_FIELD, + instance.booleanNode(immutable))); + return this; + } + + @Override + protected String getAspectName() { + return STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; + } + + @Override + protected String getEntityType() { + return STRUCTURED_PROPERTY_ENTITY_NAME; + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/ArrayMergingTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/ArrayMergingTemplate.java index ff721e97c0e1d..9c1946cd6b6d1 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/ArrayMergingTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/ArrayMergingTemplate.java @@ -12,6 +12,8 @@ public interface ArrayMergingTemplate extends Template { + static final String UNIT_SEPARATOR_DELIMITER = "␟"; + /** * Takes an Array field on the {@link RecordTemplate} subtype along with a set of key fields to * transform into a map Avoids producing side effects by copying nodes, use resulting node and not @@ -39,7 +41,15 @@ default JsonNode arrayFieldToMap( JsonNode nodeClone = node.deepCopy(); if (!keyFields.isEmpty()) { for (String keyField : keyFields) { - String key = node.get(keyField).asText(); + String key; + // if the keyField has a unit separator, we are working with a nested key + if (keyField.contains(UNIT_SEPARATOR_DELIMITER)) { + String[] keyParts = keyField.split(UNIT_SEPARATOR_DELIMITER); + JsonNode keyObject = node.get(keyParts[0]); + key = keyObject.get(keyParts[1]).asText(); + } else { + key = node.get(keyField).asText(); + } keyValue = keyValue.get(key) == null ? (ObjectNode) keyValue.set(key, instance.objectNode()).get(key) diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/AspectTemplateEngine.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/AspectTemplateEngine.java index 0818241df8155..ce36b7e77a2b1 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/AspectTemplateEngine.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/AspectTemplateEngine.java @@ -8,10 +8,12 @@ import static com.linkedin.metadata.Constants.DATA_JOB_INPUT_OUTPUT_ASPECT_NAME; import static com.linkedin.metadata.Constants.DATA_PRODUCT_PROPERTIES_ASPECT_NAME; import static com.linkedin.metadata.Constants.EDITABLE_SCHEMA_METADATA_ASPECT_NAME; +import static com.linkedin.metadata.Constants.FORM_INFO_ASPECT_NAME; import static com.linkedin.metadata.Constants.GLOBAL_TAGS_ASPECT_NAME; import static com.linkedin.metadata.Constants.GLOSSARY_TERMS_ASPECT_NAME; import static com.linkedin.metadata.Constants.OWNERSHIP_ASPECT_NAME; import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; import static com.linkedin.metadata.Constants.UPSTREAM_LINEAGE_ASPECT_NAME; import com.fasterxml.jackson.core.JsonProcessingException; @@ -46,7 +48,9 @@ public class AspectTemplateEngine { DATA_JOB_INPUT_OUTPUT_ASPECT_NAME, CHART_INFO_ASPECT_NAME, DASHBOARD_INFO_ASPECT_NAME, - STRUCTURED_PROPERTIES_ASPECT_NAME) + STRUCTURED_PROPERTIES_ASPECT_NAME, + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, + FORM_INFO_ASPECT_NAME) .collect(Collectors.toSet()); private final Map> _aspectTemplateMap; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/TemplateUtil.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/TemplateUtil.java index be3fc4c1fc983..d4e94e1e82e8f 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/TemplateUtil.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/TemplateUtil.java @@ -85,7 +85,8 @@ public static JsonNode populateTopLevelKeys(JsonNode transformedNode, JsonPatch // Skip first as it will always be blank due to path starting with / for (int i = 1; i < endIdx; i++) { if (parent.get(keys[i]) == null) { - ((ObjectNode) parent).set(keys[i], instance.objectNode()); + String decodedKey = decodeValue(keys[i]); + ((ObjectNode) parent).set(decodedKey, instance.objectNode()); } parent = parent.get(keys[i]); } @@ -93,4 +94,9 @@ public static JsonNode populateTopLevelKeys(JsonNode transformedNode, JsonPatch return transformedNodeClone; } + + /** Simply decode a JSON-patch encoded value * */ + private static String decodeValue(String value) { + return value.replace("~1", "/").replace("~0", "~"); + } } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/form/FormInfoTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/form/FormInfoTemplate.java new file mode 100644 index 0000000000000..e9288ddd889d4 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/form/FormInfoTemplate.java @@ -0,0 +1,84 @@ +package com.linkedin.metadata.aspect.patch.template.form; + +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.form.FormInfo; +import com.linkedin.metadata.aspect.patch.template.CompoundKeyTemplate; +import java.util.Collections; +import javax.annotation.Nonnull; + +public class FormInfoTemplate extends CompoundKeyTemplate { + + private static final String PROMPTS_FIELD_NAME = "prompts"; + private static final String PROMPT_ID_FIELD_NAME = "id"; + private static final String ACTORS_FIELD_NAME = "actors"; + private static final String USERS_FIELD_NAME = "users"; + private static final String GROUPS_FIELD_NAME = "groups"; + + @Override + public FormInfo getSubtype(RecordTemplate recordTemplate) throws ClassCastException { + if (recordTemplate instanceof FormInfo) { + return (FormInfo) recordTemplate; + } + throw new ClassCastException("Unable to cast RecordTemplate to FormInfo"); + } + + @Override + public Class getTemplateType() { + return FormInfo.class; + } + + @Nonnull + @Override + public FormInfo getDefault() { + FormInfo formInfo = new FormInfo(); + formInfo.setName(""); + + return formInfo; + } + + @Nonnull + @Override + public JsonNode transformFields(JsonNode baseNode) { + JsonNode transformedNode = + arrayFieldToMap( + baseNode, PROMPTS_FIELD_NAME, Collections.singletonList(PROMPT_ID_FIELD_NAME)); + + JsonNode actors = transformedNode.get(ACTORS_FIELD_NAME); + if (actors == null) { + actors = instance.objectNode(); + } + + JsonNode transformedActorsNode = + arrayFieldToMap(actors, USERS_FIELD_NAME, Collections.emptyList()); + transformedActorsNode = + arrayFieldToMap(transformedActorsNode, GROUPS_FIELD_NAME, Collections.emptyList()); + ((ObjectNode) transformedNode).set(ACTORS_FIELD_NAME, transformedActorsNode); + + return transformedNode; + } + + @Nonnull + @Override + public JsonNode rebaseFields(JsonNode patched) { + JsonNode transformedNode = + transformedMapToArray( + patched, PROMPTS_FIELD_NAME, Collections.singletonList(PROMPT_ID_FIELD_NAME)); + + JsonNode actors = transformedNode.get(ACTORS_FIELD_NAME); + if (actors == null) { + actors = instance.objectNode(); + } + + JsonNode transformedActorsNode = + transformedMapToArray(actors, USERS_FIELD_NAME, Collections.emptyList()); + transformedActorsNode = + transformedMapToArray(transformedActorsNode, GROUPS_FIELD_NAME, Collections.emptyList()); + ((ObjectNode) transformedNode).set(ACTORS_FIELD_NAME, transformedActorsNode); + + return transformedNode; + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/structuredproperty/StructuredPropertyDefinitionTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/structuredproperty/StructuredPropertyDefinitionTemplate.java new file mode 100644 index 0000000000000..2a0fa4950e157 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/structuredproperty/StructuredPropertyDefinitionTemplate.java @@ -0,0 +1,86 @@ +package com.linkedin.metadata.aspect.patch.template.structuredproperty; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.linkedin.common.UrnArray; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.aspect.patch.template.CompoundKeyTemplate; +import com.linkedin.structured.StructuredPropertyDefinition; +import java.util.Collections; +import javax.annotation.Nonnull; + +public class StructuredPropertyDefinitionTemplate + extends CompoundKeyTemplate { + + private static final String ENTITY_TYPES_FIELD_NAME = "entityTypes"; + private static final String ALLOWED_VALUES_FIELD_NAME = "allowedValues"; + private static final String VALUE_FIELD_NAME = "value"; + private static final String UNIT_SEPARATOR_DELIMITER = "␟"; + + @Override + public StructuredPropertyDefinition getSubtype(RecordTemplate recordTemplate) + throws ClassCastException { + if (recordTemplate instanceof StructuredPropertyDefinition) { + return (StructuredPropertyDefinition) recordTemplate; + } + throw new ClassCastException("Unable to cast RecordTemplate to StructuredPropertyDefinition"); + } + + @Override + public Class getTemplateType() { + return StructuredPropertyDefinition.class; + } + + @Nonnull + @Override + public StructuredPropertyDefinition getDefault() { + StructuredPropertyDefinition definition = new StructuredPropertyDefinition(); + definition.setQualifiedName(""); + definition.setValueType(UrnUtils.getUrn("urn:li:dataType:datahub.string")); + definition.setEntityTypes(new UrnArray()); + + return definition; + } + + @Nonnull + @Override + public JsonNode transformFields(JsonNode baseNode) { + JsonNode transformedNode = + arrayFieldToMap(baseNode, ENTITY_TYPES_FIELD_NAME, Collections.emptyList()); + + if (transformedNode.get(ALLOWED_VALUES_FIELD_NAME) == null) { + return transformedNode; + } + + // allowedValues has a nested key - value.string or value.number depending on type. Mapping + // needs this nested key + JsonNode allowedValues = transformedNode.get(ALLOWED_VALUES_FIELD_NAME); + if (((ArrayNode) allowedValues).size() > 0) { + JsonNode allowedValue = ((ArrayNode) allowedValues).get(0); + JsonNode value = allowedValue.get(VALUE_FIELD_NAME); + String secondaryKeyName = value.fieldNames().next(); + return arrayFieldToMap( + transformedNode, + ALLOWED_VALUES_FIELD_NAME, + Collections.singletonList( + VALUE_FIELD_NAME + UNIT_SEPARATOR_DELIMITER + secondaryKeyName)); + } + + return arrayFieldToMap( + transformedNode, ALLOWED_VALUES_FIELD_NAME, Collections.singletonList(VALUE_FIELD_NAME)); + } + + @Nonnull + @Override + public JsonNode rebaseFields(JsonNode patched) { + JsonNode patchedNode = + transformedMapToArray(patched, ENTITY_TYPES_FIELD_NAME, Collections.emptyList()); + + if (patchedNode.get(ALLOWED_VALUES_FIELD_NAME) == null) { + return patchedNode; + } + return transformedMapToArray( + patchedNode, ALLOWED_VALUES_FIELD_NAME, Collections.singletonList(VALUE_FIELD_NAME)); + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginFactory.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginFactory.java index c812aea0c55d7..bcf9108320842 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginFactory.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginFactory.java @@ -16,6 +16,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.function.BiFunction; import java.util.function.Function; @@ -48,17 +49,72 @@ public static PluginFactory merge( PluginFactory b, @Nullable BiFunction, PluginFactory> pluginFactoryProvider) { + + if (b.isEmpty()) { + return a; + } + if (a.isEmpty()) { + return b; + } + PluginConfiguration mergedPluginConfig = PluginConfiguration.merge(a.pluginConfiguration, b.pluginConfiguration); List mergedClassLoaders = Stream.concat(a.getClassLoaders().stream(), b.getClassLoaders().stream()) .collect(Collectors.toList()); - if (pluginFactoryProvider != null) { - return pluginFactoryProvider.apply(mergedPluginConfig, mergedClassLoaders); - } else { - return PluginFactory.withCustomClasspath(mergedPluginConfig, mergedClassLoaders); + if (!a.hasLoadedPlugins() && !b.hasLoadedPlugins()) { + if (pluginFactoryProvider != null) { + return pluginFactoryProvider.apply(mergedPluginConfig, mergedClassLoaders); + } else { + if (mergedPluginConfig + .streamAll() + .anyMatch(config -> config.getSpring() != null && config.getSpring().isEnabled())) { + throw new IllegalStateException( + "Unexpected Spring configuration found without a provided Spring Plugin Factory"); + } + return PluginFactory.withCustomClasspath(mergedPluginConfig, mergedClassLoaders); + } } + + PluginFactory loadedA = a.hasLoadedPlugins() ? a : a.loadPlugins(); + PluginFactory loadedB = b.hasLoadedPlugins() ? b : b.loadPlugins(); + + return new PluginFactory( + mergedPluginConfig, + mergedClassLoaders, + Stream.concat( + loadedA.aspectPayloadValidators.stream() + .filter( + aPlugin -> + loadedB.pluginConfiguration.getAspectPayloadValidators().stream() + .noneMatch(bConfig -> aPlugin.getConfig().isDisabledBy(bConfig))), + loadedB.aspectPayloadValidators.stream()) + .collect(Collectors.toList()), + Stream.concat( + loadedA.mutationHooks.stream() + .filter( + aPlugin -> + loadedB.pluginConfiguration.getMutationHooks().stream() + .noneMatch(bConfig -> aPlugin.getConfig().isDisabledBy(bConfig))), + loadedB.mutationHooks.stream()) + .collect(Collectors.toList()), + Stream.concat( + loadedA.mclSideEffects.stream() + .filter( + aPlugin -> + loadedB.pluginConfiguration.getMclSideEffects().stream() + .noneMatch(bConfig -> aPlugin.getConfig().isDisabledBy(bConfig))), + loadedB.mclSideEffects.stream()) + .collect(Collectors.toList()), + Stream.concat( + loadedA.mcpSideEffects.stream() + .filter( + aPlugin -> + loadedB.pluginConfiguration.getMcpSideEffects().stream() + .noneMatch(bConfig -> aPlugin.getConfig().isDisabledBy(bConfig))), + loadedB.mcpSideEffects.stream()) + .collect(Collectors.toList())); } @Getter private final PluginConfiguration pluginConfiguration; @@ -77,14 +133,79 @@ public PluginFactory( pluginConfiguration == null ? PluginConfiguration.EMPTY : pluginConfiguration; } + public PluginFactory( + @Nullable PluginConfiguration pluginConfiguration, + @Nonnull List classLoaders, + @Nonnull List aspectPayloadValidators, + @Nonnull List mutationHooks, + @Nonnull List mclSideEffects, + @Nonnull List mcpSideEffects) { + this.classLoaders = classLoaders; + this.pluginConfiguration = + pluginConfiguration == null ? PluginConfiguration.EMPTY : pluginConfiguration; + this.aspectPayloadValidators = applyDisable(aspectPayloadValidators); + this.mutationHooks = applyDisable(mutationHooks); + this.mclSideEffects = applyDisable(mclSideEffects); + this.mcpSideEffects = applyDisable(mcpSideEffects); + } + public PluginFactory loadPlugins() { - this.aspectPayloadValidators = buildAspectPayloadValidators(this.pluginConfiguration); - this.mutationHooks = buildMutationHooks(this.pluginConfiguration); - this.mclSideEffects = buildMCLSideEffects(this.pluginConfiguration); - this.mcpSideEffects = buildMCPSideEffects(this.pluginConfiguration); + if (this.aspectPayloadValidators != null + || this.mutationHooks != null + || this.mclSideEffects != null + || this.mcpSideEffects != null) { + log.error("Plugins are already loaded. Re-building plugins will be skipped."); + } else { + this.aspectPayloadValidators = buildAspectPayloadValidators(this.pluginConfiguration); + this.mutationHooks = buildMutationHooks(this.pluginConfiguration); + this.mclSideEffects = buildMCLSideEffects(this.pluginConfiguration); + this.mcpSideEffects = buildMCPSideEffects(this.pluginConfiguration); + logSummary( + Stream.of( + this.aspectPayloadValidators, + this.mutationHooks, + this.mclSideEffects, + this.mcpSideEffects) + .flatMap(List::stream) + .collect(Collectors.toList())); + } return this; } + public boolean isEmpty() { + return this.pluginConfiguration.isEmpty() + && Optional.ofNullable(this.aspectPayloadValidators).map(List::isEmpty).orElse(true) + && Optional.ofNullable(this.mutationHooks).map(List::isEmpty).orElse(true) + && Optional.ofNullable(this.mclSideEffects).map(List::isEmpty).orElse(true) + && Optional.ofNullable(this.mcpSideEffects).map(List::isEmpty).orElse(true); + } + + public boolean hasLoadedPlugins() { + return Stream.of( + this.aspectPayloadValidators, + this.mutationHooks, + this.mcpSideEffects, + this.mcpSideEffects) + .anyMatch(Objects::nonNull); + } + + private void logSummary(List pluginSpecs) { + if (!pluginSpecs.isEmpty()) { + log.info( + "Enabled {} plugins. {}", + pluginSpecs.size(), + pluginSpecs.stream() + .map( + v -> + String.join( + ", ", + Collections.singletonList( + String.format("%s", v.getConfig().getClassName())))) + .sorted() + .collect(Collectors.toList())); + } + } + /** * Memory intensive operation because of the size of the jars. Limit packages, classes scanned, * cache results diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/config/PluginConfiguration.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/config/PluginConfiguration.java index e9494c49a9efb..5c086982557a3 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/config/PluginConfiguration.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/config/PluginConfiguration.java @@ -27,6 +27,13 @@ public class PluginConfiguration { public static PluginConfiguration EMPTY = new PluginConfiguration(); + public boolean isEmpty() { + return aspectPayloadValidators.isEmpty() + && mutationHooks.isEmpty() + && mclSideEffects.isEmpty() + && mcpSideEffects.isEmpty(); + } + public static PluginConfiguration merge(PluginConfiguration a, PluginConfiguration b) { return new PluginConfiguration( Stream.concat( diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffect.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffect.java index 845f967c0a528..52920d8c6f396 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffect.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffect.java @@ -2,6 +2,8 @@ import com.linkedin.metadata.aspect.RetrieverContext; import com.linkedin.metadata.aspect.batch.ChangeMCP; +import com.linkedin.metadata.aspect.batch.MCLItem; +import com.linkedin.metadata.aspect.batch.MCPItem; import com.linkedin.metadata.aspect.plugins.PluginSpec; import java.util.Collection; import java.util.function.BiFunction; @@ -28,6 +30,40 @@ public final Stream apply( retrieverContext); } + /** + * Apply MCP Side Effects after commit. + * + * @param mclItems MCL items generated by MCP commit. + * @param retrieverContext accessors for aspect and graph data + * @return additional MCPs + */ + public final Stream postApply( + Collection mclItems, @Nonnull RetrieverContext retrieverContext) { + return postMCPSideEffect( + mclItems.stream() + .filter(item -> shouldApply(item.getChangeType(), item.getUrn(), item.getAspectSpec())) + .collect(Collectors.toList()), + retrieverContext); + } + + /** + * Generate additional MCPs during the transaction of the given MCPs + * + * @param changeMCPS MCPs being committed + * @param retrieverContext accessors for aspect and graph data + * @return additional MCPs + */ protected abstract Stream applyMCPSideEffect( Collection changeMCPS, @Nonnull RetrieverContext retrieverContext); + + /** + * Generate additional MCPs after the transaction of an MCP. This task will not block the + * production of the MCL for downstream processing. + * + * @param mclItems MCL items generated from committing the MCP + * @param retrieverContext accessors for aspect and graph data + * @return additional MCPs + */ + protected abstract Stream postMCPSideEffect( + Collection mclItems, @Nonnull RetrieverContext retrieverContext); } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectValidationException.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectValidationException.java index f83642c5eed9e..dd8798ee89ae6 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectValidationException.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectValidationException.java @@ -1,12 +1,16 @@ package com.linkedin.metadata.aspect.plugins.validation; import com.linkedin.common.urn.Urn; +import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.batch.BatchItem; import com.linkedin.util.Pair; -import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import lombok.EqualsAndHashCode; +import lombok.Value; +@Value +@EqualsAndHashCode(callSuper = false) public class AspectValidationException extends Exception { public static AspectValidationException forItem(BatchItem item, String msg) { @@ -14,46 +18,55 @@ public static AspectValidationException forItem(BatchItem item, String msg) { } public static AspectValidationException forItem(BatchItem item, String msg, Exception e) { - return new AspectValidationException(item.getUrn(), item.getAspectName(), msg, e); + return new AspectValidationException( + item.getChangeType(), item.getUrn(), item.getAspectName(), msg, SubType.VALIDATION, e); } - @Nonnull private final Urn entityUrn; - @Nonnull private final String aspectName; - @Nullable private final String msg; + public static AspectValidationException forPrecondition(BatchItem item, String msg) { + return forPrecondition(item, msg, null); + } + + public static AspectValidationException forPrecondition(BatchItem item, String msg, Exception e) { + return new AspectValidationException( + item.getChangeType(), item.getUrn(), item.getAspectName(), msg, SubType.PRECONDITION, e); + } - public AspectValidationException(@Nonnull Urn entityUrn, @Nonnull String aspectName, String msg) { - this(entityUrn, aspectName, msg, null); + @Nonnull ChangeType changeType; + @Nonnull Urn entityUrn; + @Nonnull String aspectName; + @Nonnull SubType subType; + @Nullable String msg; + + public AspectValidationException( + @Nonnull ChangeType changeType, + @Nonnull Urn entityUrn, + @Nonnull String aspectName, + String msg, + SubType subType) { + this(changeType, entityUrn, aspectName, msg, subType, null); } public AspectValidationException( - @Nonnull Urn entityUrn, @Nonnull String aspectName, @Nonnull String msg, Exception e) { + @Nonnull ChangeType changeType, + @Nonnull Urn entityUrn, + @Nonnull String aspectName, + @Nonnull String msg, + @Nullable SubType subType, + Exception e) { super(msg, e); + this.changeType = changeType; this.entityUrn = entityUrn; this.aspectName = aspectName; this.msg = msg; + this.subType = subType != null ? subType : SubType.VALIDATION; } - public Pair getExceptionKey() { + public Pair getAspectGroup() { return Pair.of(entityUrn, aspectName); } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - AspectValidationException that = (AspectValidationException) o; - - if (!entityUrn.equals(that.entityUrn)) return false; - if (!aspectName.equals(that.aspectName)) return false; - return Objects.equals(msg, that.msg); - } - - @Override - public int hashCode() { - int result = entityUrn.hashCode(); - result = 31 * result + aspectName.hashCode(); - result = 31 * result + (msg != null ? msg.hashCode() : 0); - return result; + public static enum SubType { + VALIDATION, + PRECONDITION } } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/ValidationExceptionCollection.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/ValidationExceptionCollection.java index 559fa85cff04c..007c196156b12 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/ValidationExceptionCollection.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/ValidationExceptionCollection.java @@ -20,7 +20,7 @@ public static ValidationExceptionCollection newCollection() { } public void addException(AspectValidationException exception) { - super.computeIfAbsent(exception.getExceptionKey(), key -> new HashSet<>()).add(exception); + super.computeIfAbsent(exception.getAspectGroup(), key -> new HashSet<>()).add(exception); } public void addException(BatchItem item, String message) { diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/ConditionalWriteValidator.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/ConditionalWriteValidator.java new file mode 100644 index 0000000000000..810693c80fa13 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/ConditionalWriteValidator.java @@ -0,0 +1,225 @@ +package com.linkedin.metadata.aspect.validation; + +import com.google.common.collect.ImmutableSet; +import com.google.common.net.HttpHeaders; +import com.linkedin.common.urn.Urn; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.AspectRetriever; +import com.linkedin.metadata.aspect.RetrieverContext; +import com.linkedin.metadata.aspect.SystemAspect; +import com.linkedin.metadata.aspect.batch.BatchItem; +import com.linkedin.metadata.aspect.batch.ChangeMCP; +import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; +import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator; +import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; +import com.linkedin.metadata.aspect.plugins.validation.ValidationExceptionCollection; +import com.linkedin.util.Pair; +import java.time.Instant; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; + +/** + * Implements conditional write semantics. + * + *

1. If an aspect change request includes a SystemMetadata section which includes a value for + * nextAspectVersion, then it must match the current SystemMetadata's value for nextAspectVersion or + * prevent performing the write operation. + * + *

2. Headers are included with the MCP for the following time-based checks * If-Modified-Since * + * If-Unmodified-Since + */ +@Setter +@Getter +@Accessors(chain = true) +public class ConditionalWriteValidator extends AspectPayloadValidator { + public static final String UNVERSIONED_ASPECT_VERSION = "-1"; + public static final long DEFAULT_LAST_MODIFIED_TIME = Long.MIN_VALUE; + public static final String HTTP_HEADER_IF_VERSION_MATCH = "If-Version-Match"; + public static final Set CREATE_CHANGE_TYPES = + ImmutableSet.of(ChangeType.CREATE, ChangeType.CREATE_ENTITY); + + @Nonnull private AspectPluginConfig config; + + private static boolean hasTimePrecondition(ChangeMCP item) { + return item.getHeader(HttpHeaders.IF_MODIFIED_SINCE).isPresent() + || item.getHeader(HttpHeaders.IF_UNMODIFIED_SINCE).isPresent(); + } + + private static boolean hasVersionPrecondition(ChangeMCP item) { + return item.getHeader(HTTP_HEADER_IF_VERSION_MATCH).isPresent(); + } + + private static boolean isApplicableFilter(ChangeMCP item) { + if (ChangeType.RESTATE.equals(item.getChangeType())) { + return false; + } + + return hasTimePrecondition(item) || hasVersionPrecondition(item); + } + + @Override + protected Stream validatePreCommitAspects( + @Nonnull Collection changeMCPs, @Nonnull RetrieverContext retrieverContext) { + ValidationExceptionCollection exceptions = ValidationExceptionCollection.newCollection(); + AspectRetriever aspectRetriever = retrieverContext.getAspectRetriever(); + + List applicableMCPs = + changeMCPs.stream() + .filter(ConditionalWriteValidator::isApplicableFilter) + .collect(Collectors.toList()); + + // Batch lookup missing data + Map> missingDataUrnAspects = + applicableMCPs.stream() + // create change types not expected to have previous data + .filter(item -> !CREATE_CHANGE_TYPES.contains(item.getChangeType())) + .filter(item -> item.getPreviousSystemAspect() == null) + .collect( + Collectors.groupingBy( + ChangeMCP::getUrn, + Collectors.mapping(ChangeMCP::getAspectName, Collectors.toSet()))); + final Map> resolvedData = + aspectRetriever.getLatestSystemAspects(missingDataUrnAspects); + + for (ChangeMCP item : applicableMCPs) { + // Validate aspect version precondition + if (hasVersionPrecondition(item)) { + item.getHeader(HTTP_HEADER_IF_VERSION_MATCH) + .flatMap( + headerValue -> + validateVersionPrecondition( + item, Pair.of(HTTP_HEADER_IF_VERSION_MATCH, headerValue), resolvedData)) + .ifPresent(exceptions::addException); + } + + // Validate modified time conditional + if (hasTimePrecondition(item)) { + item.getHeader(HttpHeaders.IF_MODIFIED_SINCE) + .flatMap( + headerValue -> + validateTimePrecondition( + item, Pair.of(HttpHeaders.IF_MODIFIED_SINCE, headerValue), resolvedData)) + .ifPresent(exceptions::addException); + item.getHeader(HttpHeaders.IF_UNMODIFIED_SINCE) + .flatMap( + headerValue -> + validateTimePrecondition( + item, Pair.of(HttpHeaders.IF_UNMODIFIED_SINCE, headerValue), resolvedData)) + .ifPresent(exceptions::addException); + } + } + + return exceptions.streamAllExceptions(); + } + + private static Optional validateVersionPrecondition( + ChangeMCP item, + Pair header, + Map> resolvedData) { + final String actualAspectVersion; + switch (item.getChangeType()) { + case CREATE: + case CREATE_ENTITY: + actualAspectVersion = UNVERSIONED_ASPECT_VERSION; + break; + default: + actualAspectVersion = + resolvePreviousSystemAspect(item, resolvedData) + .map( + prevSystemAspect -> { + if (prevSystemAspect.getSystemMetadataVersion().isPresent()) { + return String.valueOf(prevSystemAspect.getSystemMetadataVersion().get()); + } else { + return String.valueOf(Math.max(1, prevSystemAspect.getVersion())); + } + }) + .orElse(UNVERSIONED_ASPECT_VERSION); + break; + } + + if (!header.getSecond().equals(actualAspectVersion)) { + return Optional.of( + AspectValidationException.forPrecondition( + item, + String.format( + "Expected version %s, actual version %s", + header.getSecond(), actualAspectVersion))); + } + + return Optional.empty(); + } + + private static Optional validateTimePrecondition( + ChangeMCP item, + Pair header, + Map> resolvedData) { + + final long lastModifiedTimeMs; + switch (item.getChangeType()) { + case CREATE: + case CREATE_ENTITY: + lastModifiedTimeMs = DEFAULT_LAST_MODIFIED_TIME; + break; + default: + lastModifiedTimeMs = + resolvePreviousSystemAspect(item, resolvedData) + .map(systemAspect -> systemAspect.getAuditStamp().getTime()) + .orElse(DEFAULT_LAST_MODIFIED_TIME); + break; + } + + long headerValueEpochMs = Instant.parse(header.getValue()).toEpochMilli(); + + switch (header.getKey()) { + case HttpHeaders.IF_MODIFIED_SINCE: + return lastModifiedTimeMs > headerValueEpochMs + ? Optional.empty() + : Optional.of( + AspectValidationException.forPrecondition( + item, + String.format( + "Item last modified %s <= %s (epoch ms)", + lastModifiedTimeMs, headerValueEpochMs))); + case HttpHeaders.IF_UNMODIFIED_SINCE: + return lastModifiedTimeMs <= headerValueEpochMs + ? Optional.empty() + : Optional.of( + AspectValidationException.forPrecondition( + item, + String.format( + "Item last modified %s > %s (epoch ms)", + lastModifiedTimeMs, headerValueEpochMs))); + default: + return Optional.empty(); + } + } + + private static Optional resolvePreviousSystemAspect( + ChangeMCP item, Map> resolvedData) { + if (item.getPreviousSystemAspect() != null) { + return Optional.of(item.getPreviousSystemAspect()); + } + if (resolvedData.getOrDefault(item.getUrn(), Collections.emptyMap()).get(item.getAspectName()) + != null) { + return Optional.of(resolvedData.get(item.getUrn()).get(item.getAspectName())); + } + return Optional.empty(); + } + + @Override + protected Stream validateProposedAspects( + @Nonnull Collection mcpItems, + @Nonnull RetrieverContext retrieverContext) { + return Stream.empty(); + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/entity/SearchRetriever.java b/entity-registry/src/main/java/com/linkedin/metadata/entity/SearchRetriever.java new file mode 100644 index 0000000000000..eaa106b8d1f63 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/entity/SearchRetriever.java @@ -0,0 +1,24 @@ +package com.linkedin.metadata.entity; + +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.ScrollResult; +import java.util.List; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public interface SearchRetriever { + /** + * Returns search results for the given entities, filtered and sorted. + * + * @param entities list of entities to search + * @param filters filters to apply + * @param scrollId pagination token + * @param count size of a page + * @return result of the search + */ + ScrollResult scroll( + @Nonnull List entities, + @Nullable Filter filters, + @Nullable String scrollId, + int count); +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java b/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java index d8fb67c60469b..885be5e00b945 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java @@ -1,33 +1,44 @@ package com.linkedin.metadata.models; import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME; import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD; import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD_PREFIX; import com.google.common.collect.ImmutableSet; import com.linkedin.common.Status; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.GetMode; +import com.linkedin.entity.Aspect; import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.Criterion; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.structured.PrimitivePropertyValue; import com.linkedin.structured.StructuredProperties; +import com.linkedin.structured.StructuredPropertyDefinition; import com.linkedin.structured.StructuredPropertyValueAssignment; import com.linkedin.structured.StructuredPropertyValueAssignmentArray; import com.linkedin.util.Pair; import java.sql.Date; import java.time.format.DateTimeParseException; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class StructuredPropertyUtils { private StructuredPropertyUtils() {} @@ -35,26 +46,169 @@ private StructuredPropertyUtils() {} static final Date MIN_DATE = Date.valueOf("1000-01-01"); static final Date MAX_DATE = Date.valueOf("9999-12-31"); + public static LogicalValueType getLogicalValueType( + StructuredPropertyDefinition structuredPropertyDefinition) { + return getLogicalValueType(structuredPropertyDefinition.getValueType()); + } + + public static LogicalValueType getLogicalValueType(@Nullable Urn valueType) { + String valueTypeId = getValueTypeId(valueType); + if ("string".equals(valueTypeId)) { + return LogicalValueType.STRING; + } else if ("date".equals(valueTypeId)) { + return LogicalValueType.DATE; + } else if ("number".equals(valueTypeId)) { + return LogicalValueType.NUMBER; + } else if ("urn".equals(valueTypeId)) { + return LogicalValueType.URN; + } else if ("rich_text".equals(valueTypeId)) { + return LogicalValueType.RICH_TEXT; + } + return LogicalValueType.UNKNOWN; + } + + @Nullable + public static String getValueTypeId(@Nullable final Urn valueType) { + if (valueType != null) { + String valueTypeId = valueType.getId(); + if (valueTypeId.startsWith("datahub.")) { + valueTypeId = valueTypeId.split("\\.")[1]; + } + return valueTypeId.toLowerCase(); + } else { + return null; + } + } + /** - * Sanitizes fully qualified name for use in an ElasticSearch field name Replaces . and " " + * Given a structured property input field or facet name, return a valid structured property facet + * name + * + * @param fieldOrFacetName input name + * @param aspectRetriever aspect retriever + * @return guranteed facet name + */ + public static Optional toStructuredPropertyFacetName( + @Nonnull String fieldOrFacetName, @Nullable AspectRetriever aspectRetriever) { + return lookupDefinitionFromFilterOrFacetName(fieldOrFacetName, aspectRetriever) + .map( + urnDefinition -> { + switch (getLogicalValueType(urnDefinition.getSecond())) { + case DATE: + case NUMBER: + return STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX + + StructuredPropertyUtils.toElasticsearchFieldName( + urnDefinition.getFirst(), urnDefinition.getSecond()); + default: + return STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX + + StructuredPropertyUtils.toElasticsearchFieldName( + urnDefinition.getFirst(), urnDefinition.getSecond()) + + ".keyword"; + } + }); + } + + /** + * Lookup structured property definition given the name used for the field in APIs such as a + * search filter or aggregation query facet name. + * + * @param fieldOrFacetName the field name used in a filter or facet name in an aggregation query + * @param aspectRetriever method to look up the definition aspect + * @return the structured property definition if found + */ + public static Optional> + lookupDefinitionFromFilterOrFacetName( + @Nonnull String fieldOrFacetName, @Nullable AspectRetriever aspectRetriever) { + if (fieldOrFacetName.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD + ".")) { + String fqn = + fieldOrFacetName + .substring(STRUCTURED_PROPERTY_MAPPING_FIELD.length() + 1) + .replace(".keyword", "") + .replace(".delimited", ""); + Urn urn = toURNFromFQN(fqn); + Map> result = + Objects.requireNonNull(aspectRetriever) + .getLatestAspectObjects( + Collections.singleton(urn), + Collections.singleton(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)); + Optional definition = + Optional.ofNullable( + result + .getOrDefault(urn, Collections.emptyMap()) + .getOrDefault(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, null)); + return definition.map( + definitonAspect -> + Pair.of(urn, new StructuredPropertyDefinition(definitonAspect.data()))); + } + return Optional.empty(); + } + + /** + * Given the structured property definition extract the Elasticsearch field name with nesting and + * character replacement. + * + *

Sanitizes fully qualified name for use in an ElasticSearch field name Replaces `.` * characters * - * @param fullyQualifiedName The original fully qualified name of the property + * @param definition The structured property definition * @return The sanitized version that can be used as a field name */ - public static String sanitizeStructuredPropertyFQN(@Nonnull String fullyQualifiedName) { - if (fullyQualifiedName.contains(" ")) { + public static String toElasticsearchFieldName( + @Nonnull Urn propertyUrn, @Nullable StructuredPropertyDefinition definition) { + String qualifiedName = definition != null ? definition.getQualifiedName() : propertyUrn.getId(); + + if (qualifiedName.contains(" ")) { throw new IllegalArgumentException( "Fully qualified structured property name cannot contain spaces"); } - return fullyQualifiedName.replace('.', '_'); + if (definition != null && definition.getVersion(GetMode.NULL) != null) { + // includes type suffix + return String.join( + ".", + STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD, + definition.getQualifiedName().replace('.', '_'), + definition.getVersion(), + getLogicalValueType(definition).name().toLowerCase()); + } else { + // un-typed property + return qualifiedName.replace('.', '_'); + } + } + + /** + * Return an elasticsearch type from structured property type + * + * @param fieldName filter or facet field name + * @param aspectRetriever aspect retriever + * @return elasticsearch type + */ + public static Set toElasticsearchFieldType( + @Nonnull String fieldName, @Nullable AspectRetriever aspectRetriever) { + LogicalValueType logicalValueType = + lookupDefinitionFromFilterOrFacetName(fieldName, aspectRetriever) + .map(definition -> getLogicalValueType(definition.getValue())) + .orElse(LogicalValueType.STRING); + + switch (logicalValueType) { + case NUMBER: + return Collections.singleton("double"); + case DATE: + return Collections.singleton("long"); + case RICH_TEXT: + return Collections.singleton("text"); + case UNKNOWN: + case STRING: + case URN: + default: + return Collections.singleton("keyword"); + } } public static void validateStructuredPropertyFQN( @Nonnull Collection fullyQualifiedNames, @Nonnull AspectRetriever aspectRetriever) { Set structuredPropertyUrns = fullyQualifiedNames.stream() - .map(StructuredPropertyUtils::toURNFromFieldName) + .map(StructuredPropertyUtils::toURNFromFQN) .collect(Collectors.toSet()); Set removedUrns = getRemovedUrns(structuredPropertyUrns, aspectRetriever); if (!removedUrns.isEmpty()) { @@ -63,13 +217,19 @@ public static void validateStructuredPropertyFQN( } } - public static Urn toURNFromFieldName(@Nonnull String fieldName) { + /** + * Given a Structured Property fqn, calculate the expected URN + * + * @param fqn structured property's fqn + * @return the expected structured property urn + */ + private static Urn toURNFromFQN(@Nonnull String fqn) { return UrnUtils.getUrn( - String.join(":", "urn:li", STRUCTURED_PROPERTY_ENTITY_NAME, fieldName.replace('_', '.'))); + String.join(":", "urn:li", STRUCTURED_PROPERTY_ENTITY_NAME, fqn.replace('_', '.'))); } public static void validateFilter( - @Nullable Filter filter, @Nonnull AspectRetriever aspectRetriever) { + @Nullable Filter filter, @Nullable AspectRetriever aspectRetriever) { if (filter == null) { return; @@ -80,7 +240,7 @@ public static void validateFilter( if (filter.getCriteria() != null) { for (Criterion c : filter.getCriteria()) { if (c.getField().startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX)) { - fieldNames.add(c.getField().substring(STRUCTURED_PROPERTY_MAPPING_FIELD.length() + 1)); + fieldNames.add(stripStructuredPropertyPrefix(c.getField())); } } } @@ -89,15 +249,24 @@ public static void validateFilter( for (ConjunctiveCriterion cc : filter.getOr()) { for (Criterion c : cc.getAnd()) { if (c.getField().startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX)) { - fieldNames.add(c.getField().substring(STRUCTURED_PROPERTY_MAPPING_FIELD.length() + 1)); + fieldNames.add(stripStructuredPropertyPrefix(c.getField())); } } } } if (!fieldNames.isEmpty()) { - validateStructuredPropertyFQN(fieldNames, aspectRetriever); + validateStructuredPropertyFQN(fieldNames, Objects.requireNonNull(aspectRetriever)); + } + } + + private static String stripStructuredPropertyPrefix(String s) { + if (s.startsWith(STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD_PREFIX)) { + return s.substring(STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD.length() + 1).split("[.]")[0]; + } else if (s.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX)) { + return s.substring(STRUCTURED_PROPERTY_MAPPING_FIELD.length() + 1).split("[.]")[0]; } + return s; } public static Date toDate(PrimitivePropertyValue value) throws DateTimeParseException { @@ -120,7 +289,7 @@ public static boolean isValidDate(PrimitivePropertyValue value) { return date.compareTo(MIN_DATE) >= 0 && date.compareTo(MAX_DATE) <= 0; } - private static Set getRemovedUrns(Set urns, AspectRetriever aspectRetriever) { + private static Set getRemovedUrns(Set urns, @Nonnull AspectRetriever aspectRetriever) { return aspectRetriever .getLatestAspectObjects(urns, ImmutableSet.of(STATUS_ASPECT_NAME)) .entrySet() diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java index 22aeddb6ac65f..c60f89c510cd7 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java @@ -20,6 +20,8 @@ import com.linkedin.metadata.aspect.patch.template.dataset.DatasetPropertiesTemplate; import com.linkedin.metadata.aspect.patch.template.dataset.EditableSchemaMetadataTemplate; import com.linkedin.metadata.aspect.patch.template.dataset.UpstreamLineageTemplate; +import com.linkedin.metadata.aspect.patch.template.form.FormInfoTemplate; +import com.linkedin.metadata.aspect.patch.template.structuredproperty.StructuredPropertyDefinitionTemplate; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.DefaultEntitySpec; import com.linkedin.metadata.models.EntitySpec; @@ -87,6 +89,9 @@ private AspectTemplateEngine populateTemplateEngine(Map aspe aspectSpecTemplateMap.put(DATA_JOB_INPUT_OUTPUT_ASPECT_NAME, new DataJobInputOutputTemplate()); aspectSpecTemplateMap.put( STRUCTURED_PROPERTIES_ASPECT_NAME, new StructuredPropertiesTemplate()); + aspectSpecTemplateMap.put( + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, new StructuredPropertyDefinitionTemplate()); + aspectSpecTemplateMap.put(FORM_INFO_ASPECT_NAME, new FormInfoTemplate()); return new AspectTemplateEngine(aspectSpecTemplateMap); } diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java index 8c7bfc98b2673..5042c35d2f5d4 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java @@ -4,6 +4,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; @@ -254,4 +255,63 @@ public void testLargePatchStandard() throws Exception { assertEquals(result.getUpstreams().size(), 187, "Expected 1 less upstream"); assertEquals(result.getFineGrainedLineages().size(), 607); } + + @Test + public void testPatchWithFieldWithForwardSlash() throws JsonProcessingException { + + String downstreamUrn = + "/fineGrainedLineages/CREATE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c1)"; + String unescapedUpstreamUrn = + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),slash/column)"; + String escapedUpstreamUrn = + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),slash~1column)"; + String lineagePath = downstreamUrn + "//" + escapedUpstreamUrn; + + UpstreamLineageTemplate upstreamLineageTemplate = new UpstreamLineageTemplate(); + UpstreamLineage upstreamLineage = upstreamLineageTemplate.getDefault(); + JsonPatchBuilder jsonPatchBuilder = Json.createPatchBuilder(); + + JsonObjectBuilder fineGrainedLineageNode = Json.createObjectBuilder(); + JsonValue upstreamConfidenceScore = Json.createValue(1.0f); + fineGrainedLineageNode.add("confidenceScore", upstreamConfidenceScore); + + jsonPatchBuilder.add(lineagePath, fineGrainedLineageNode.build()); + + // Initial population test + UpstreamLineage result = + upstreamLineageTemplate.applyPatch(upstreamLineage, jsonPatchBuilder.build()); + + assertEquals( + result.getFineGrainedLineages().get(0).getUpstreams().get(0).toString(), + unescapedUpstreamUrn); + } + + @Test + public void testPatchWithFieldWithTilde() throws JsonProcessingException { + + String downstreamUrn = + "/fineGrainedLineages/CREATE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c1)"; + String unescapedUpstreamUrn = + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),tilde~column)"; + String escapedUpstreamUrn = + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_2,PROD),tilde~0column)"; + String lineagePath = downstreamUrn + "//" + escapedUpstreamUrn; + + UpstreamLineageTemplate upstreamLineageTemplate = new UpstreamLineageTemplate(); + UpstreamLineage upstreamLineage = upstreamLineageTemplate.getDefault(); + JsonPatchBuilder jsonPatchBuilder = Json.createPatchBuilder(); + + JsonObjectBuilder fineGrainedLineageNode = Json.createObjectBuilder(); + JsonValue upstreamConfidenceScore = Json.createValue(1.0f); + fineGrainedLineageNode.add("confidenceScore", upstreamConfidenceScore); + + jsonPatchBuilder.add(lineagePath, fineGrainedLineageNode.build()); + + // Initial population test + UpstreamLineage result = + upstreamLineageTemplate.applyPatch(upstreamLineage, jsonPatchBuilder.build()); + assertEquals( + result.getFineGrainedLineages().get(0).getUpstreams().get(0).toString(), + unescapedUpstreamUrn); + } } diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/PluginsTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/PluginsTest.java index a9f903f4b7017..cecf21849f3aa 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/PluginsTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/PluginsTest.java @@ -226,4 +226,60 @@ public void tripleMergeWithDisabled() throws EntityRegistryException { .count(), 0); } + + @Test + public void testEmptyMerges() throws EntityRegistryException { + ConfigEntityRegistry configEntityRegistry1 = + new ConfigEntityRegistry( + TestEntityProfile.class.getClassLoader().getResourceAsStream(REGISTRY_FILE_1)); + ConfigEntityRegistry emptyEntityRegistry = + new ConfigEntityRegistry( + TestEntityProfile.class.getClassLoader().getResourceAsStream(REGISTRY_FILE_2), + (config, classLoaders) -> PluginFactory.empty()); + + MergedEntityRegistry mergedEntityRegistry = new MergedEntityRegistry(configEntityRegistry1); + mergedEntityRegistry.apply(emptyEntityRegistry); + assertEquals(mergedEntityRegistry.getPluginFactory(), configEntityRegistry1.getPluginFactory()); + + MergedEntityRegistry mergedEntityRegistry2 = new MergedEntityRegistry(emptyEntityRegistry); + mergedEntityRegistry2.apply(configEntityRegistry1); + assertEquals( + mergedEntityRegistry2.getPluginFactory(), configEntityRegistry1.getPluginFactory()); + } + + @Test + public void testUnloadedMerge() throws EntityRegistryException { + ConfigEntityRegistry configEntityRegistry1 = + new ConfigEntityRegistry( + TestEntityProfile.class.getClassLoader().getResourceAsStream(REGISTRY_FILE_1), + (config, classLoaders) -> new PluginFactory(config, classLoaders)); + ConfigEntityRegistry configEntityRegistry2 = + new ConfigEntityRegistry( + TestEntityProfile.class.getClassLoader().getResourceAsStream(REGISTRY_FILE_2), + (config, classLoaders) -> new PluginFactory(config, classLoaders)); + + MergedEntityRegistry mergedEntityRegistry = new MergedEntityRegistry(configEntityRegistry1); + mergedEntityRegistry.apply(configEntityRegistry2); + + assertEquals( + mergedEntityRegistry.getAllAspectPayloadValidators().stream() + .filter(p -> p.getConfig().getSupportedOperations().contains("DELETE")) + .count(), + 1); + assertEquals( + mergedEntityRegistry.getAllMutationHooks().stream() + .filter(p -> p.getConfig().getSupportedOperations().contains("DELETE")) + .count(), + 1); + assertEquals( + mergedEntityRegistry.getAllMCLSideEffects().stream() + .filter(p -> p.getConfig().getSupportedOperations().contains("DELETE")) + .count(), + 1); + assertEquals( + mergedEntityRegistry.getAllMCPSideEffects().stream() + .filter(p -> p.getConfig().getSupportedOperations().contains("DELETE")) + .count(), + 1); + } } diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffectTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffectTest.java index 8e877d1d23aad..1137c7c55880f 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffectTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffectTest.java @@ -7,6 +7,8 @@ import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.RetrieverContext; import com.linkedin.metadata.aspect.batch.ChangeMCP; +import com.linkedin.metadata.aspect.batch.MCLItem; +import com.linkedin.metadata.aspect.batch.MCPItem; import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; import com.linkedin.metadata.models.registry.ConfigEntityRegistry; import java.util.Collection; @@ -72,5 +74,11 @@ protected Stream applyMCPSideEffect( Collection changeMCPS, @Nonnull RetrieverContext retrieverContext) { return changeMCPS.stream(); } + + @Override + protected Stream postMCPSideEffect( + Collection mclItems, @Nonnull RetrieverContext retrieverContext) { + return Stream.of(); + } } } diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/ConditionalWriteValidatorTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/ConditionalWriteValidatorTest.java new file mode 100644 index 0000000000000..652328781c3bb --- /dev/null +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/ConditionalWriteValidatorTest.java @@ -0,0 +1,1174 @@ +package com.linkedin.metadata.aspect.validators; + +import static com.linkedin.metadata.aspect.validation.ConditionalWriteValidator.HTTP_HEADER_IF_VERSION_MATCH; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; + +import com.google.common.net.HttpHeaders; +import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringMap; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.AspectRetriever; +import com.linkedin.metadata.aspect.GraphRetriever; +import com.linkedin.metadata.aspect.RetrieverContext; +import com.linkedin.metadata.aspect.batch.ChangeMCP; +import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; +import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; +import com.linkedin.metadata.aspect.validation.ConditionalWriteValidator; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.mxe.SystemMetadata; +import com.linkedin.test.metadata.aspect.TestEntityRegistry; +import com.linkedin.test.metadata.aspect.batch.TestMCP; +import com.linkedin.test.metadata.aspect.batch.TestSystemAspect; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +public class ConditionalWriteValidatorTest { + private EntityRegistry entityRegistry; + private RetrieverContext mockRetrieverContext; + private static final List supportedChangeTypes = + List.of( + ChangeType.CREATE, + ChangeType.CREATE_ENTITY, + ChangeType.DELETE, + ChangeType.UPSERT, + ChangeType.UPDATE, + ChangeType.RESTATE, + ChangeType.PATCH); + + private static final AspectPluginConfig validatorConfig = + AspectPluginConfig.builder() + .supportedOperations( + supportedChangeTypes.stream().map(Object::toString).collect(Collectors.toList())) + .className(ConditionalWriteValidator.class.getName()) + .supportedEntityAspectNames(List.of(AspectPluginConfig.EntityAspectName.ALL)) + .enabled(true) + .build(); + + @BeforeTest + public void init() { + entityRegistry = new TestEntityRegistry(); + AspectRetriever mockAspectRetriever = mock(AspectRetriever.class); + when(mockAspectRetriever.getEntityRegistry()).thenReturn(entityRegistry); + GraphRetriever mockGraphRetriever = mock(GraphRetriever.class); + mockRetrieverContext = mock(RetrieverContext.class); + when(mockRetrieverContext.getAspectRetriever()).thenReturn(mockAspectRetriever); + when(mockRetrieverContext.getGraphRetriever()).thenReturn(mockGraphRetriever); + } + + @Test + public void testNextVersionSuccess() { + ConditionalWriteValidator test = new ConditionalWriteValidator().setConfig(validatorConfig); + Urn testEntityUrn = UrnUtils.getUrn("urn:li:chart:(looker,baz1)"); + + for (ChangeType changeType : supportedChangeTypes) { + final ChangeMCP testMCP; + switch (changeType) { + case RESTATE: + case CREATE_ENTITY: + case CREATE: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "-1")) + .build(); + break; + default: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "1")) + .previousSystemAspect( + TestSystemAspect.builder() + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + // Previous / actual + .systemMetadata(new SystemMetadata().setVersion("1")) + .build()) + .build(); + break; + } + + Set exceptions = + test.validatePreCommit(List.of(testMCP), mockRetrieverContext) + .collect(Collectors.toSet()); + + assertEquals(Set.of(), exceptions, "Expected no exceptions for change type " + changeType); + } + } + + @Test + public void testNoSystemMetadataNextVersionNextVersionSuccess() { + ConditionalWriteValidator test = new ConditionalWriteValidator().setConfig(validatorConfig); + Urn testEntityUrn = UrnUtils.getUrn("urn:li:chart:(looker,baz1)"); + + for (ChangeType changeType : supportedChangeTypes) { + final ChangeMCP testMCP; + switch (changeType) { + case DELETE: + reset(mockRetrieverContext.getAspectRetriever()); + when(mockRetrieverContext + .getAspectRetriever() + .getLatestSystemAspects(eq(Map.of(testEntityUrn, Set.of("status"))))) + .thenReturn( + Map.of( + testEntityUrn, + Map.of( + "status", + TestSystemAspect.builder() + .systemMetadata(new SystemMetadata().setVersion("1")) + .build()))); + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected (cannot delete non-existent -1) + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "1")) + .build(); + break; + case CREATE: + case CREATE_ENTITY: + reset(mockRetrieverContext.getAspectRetriever()); + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "-1")) + .previousSystemAspect( + TestSystemAspect.builder() + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + // Missing previous system metadata, expect fallback to version + .version(0) + .build()) + .build(); + break; + default: + reset(mockRetrieverContext.getAspectRetriever()); + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "1")) + .previousSystemAspect( + TestSystemAspect.builder() + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + // Missing previous system metadata, expect fallback to version + .version(0) + .build()) + .build(); + break; + } + + Set exceptions = + test.validatePreCommit(List.of(testMCP), mockRetrieverContext) + .collect(Collectors.toSet()); + + assertEquals( + Set.of(), + exceptions, + String.format( + "Expected no exceptions for change type %s but found %s", changeType, exceptions)); + } + } + + @Test + public void testNoPreviousVersionsLookupSchemaMetadataNextVersionSuccess() { + ConditionalWriteValidator test = new ConditionalWriteValidator().setConfig(validatorConfig); + Urn testEntityUrn = UrnUtils.getUrn("urn:li:chart:(looker,baz1)"); + + // Prepare mock lookup based on version + reset(mockRetrieverContext.getAspectRetriever()); + when(mockRetrieverContext + .getAspectRetriever() + .getLatestSystemAspects(eq(Map.of(testEntityUrn, Set.of("status"))))) + .thenReturn( + Map.of( + testEntityUrn, + Map.of( + "status", + TestSystemAspect.builder() + .systemMetadata( + new SystemMetadata().setVersion("2")) // expected next version 2 + .build()))); + + for (ChangeType changeType : supportedChangeTypes) { + final ChangeMCP testMCP; + switch (changeType) { + case CREATE: + case CREATE_ENTITY: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected is always 1 + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "-1")) + .build(); + break; + default: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "2")) + .build(); + break; + } + + Set exceptions = + test.validatePreCommit(List.of(testMCP), mockRetrieverContext) + .collect(Collectors.toSet()); + + assertEquals(Set.of(), exceptions, "Expected no exceptions for change type " + changeType); + } + } + + @Test + public void testNoPreviousVersionsLookupVersionNextVersionSuccess() { + ConditionalWriteValidator test = new ConditionalWriteValidator().setConfig(validatorConfig); + Urn testEntityUrn = UrnUtils.getUrn("urn:li:chart:(looker,baz1)"); + + // Prepare mock lookup based on version + when(mockRetrieverContext + .getAspectRetriever() + .getLatestSystemAspects(eq(Map.of(testEntityUrn, Set.of("status"))))) + .thenReturn( + Map.of( + testEntityUrn, + Map.of( + "status", + TestSystemAspect.builder() + .version(2) // expected next version 2 + .build()))); + + for (ChangeType changeType : supportedChangeTypes) { + final ChangeMCP testMCP; + switch (changeType) { + case CREATE: + case CREATE_ENTITY: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected is always 1 + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "-1")) + .build(); + break; + default: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "2")) + .build(); + break; + } + + Set exceptions = + test.validatePreCommit(List.of(testMCP), mockRetrieverContext) + .collect(Collectors.toSet()); + + assertEquals(Set.of(), exceptions, "Expected no exceptions for change type " + changeType); + } + } + + @Test + public void testNextVersionFail() { + ConditionalWriteValidator test = new ConditionalWriteValidator().setConfig(validatorConfig); + Urn testEntityUrn = UrnUtils.getUrn("urn:li:chart:(looker,baz1)"); + + for (ChangeType changeType : supportedChangeTypes) { + final ChangeMCP testMCP; + switch (changeType) { + case DELETE: + // allow lookup of previous value + when(mockRetrieverContext + .getAspectRetriever() + .getLatestSystemAspects(Map.of(testEntityUrn, Set.of("status")))) + .thenReturn( + Map.of( + testEntityUrn, + Map.of( + "status", + TestSystemAspect.builder() + .urn(testEntityUrn) + .version(3) + .recordTemplate(new Status().setRemoved(false)) + .build()))); + // fall through + case RESTATE: + case CREATE_ENTITY: + case CREATE: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Incorrect Expected + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "2")) + .build(); + break; + default: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Incorrect Expected + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "2")) + .previousSystemAspect( + TestSystemAspect.builder() + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + // Previous / actual + .systemMetadata(new SystemMetadata().setVersion("3")) + .build()) + .build(); + break; + } + + Set exceptions = + test.validatePreCommit(List.of(testMCP), mockRetrieverContext) + .collect(Collectors.toSet()); + + switch (changeType) { + case RESTATE: + assertEquals(Set.of(), exceptions, "Expected no exception for change type RESTATE"); + break; + case CREATE: + case CREATE_ENTITY: + assertEquals(exceptions.size(), 1, "Expected exception for change type " + changeType); + assertEquals( + exceptions.stream().findFirst().get().getMessage(), + "Expected version 2, actual version -1"); + break; + default: + assertEquals(exceptions.size(), 1, "Expected exception for change type " + changeType); + assertEquals( + exceptions.stream().findFirst().get().getMessage(), + "Expected version 2, actual version 3", + "for changeType:" + changeType); + break; + } + } + } + + @Test + public void testNoSystemMetadataNextVersionNextVersionFail() { + ConditionalWriteValidator test = new ConditionalWriteValidator().setConfig(validatorConfig); + Urn testEntityUrn = UrnUtils.getUrn("urn:li:chart:(looker,baz1)"); + + for (ChangeType changeType : supportedChangeTypes) { + final ChangeMCP testMCP; + switch (changeType) { + default: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "2")) + .previousSystemAspect( + TestSystemAspect.builder() + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + // Missing previous system metadata, expect fallback to version + .version(3) + .build()) + .build(); + break; + } + + Set exceptions = + test.validatePreCommit(List.of(testMCP), mockRetrieverContext) + .collect(Collectors.toSet()); + + switch (changeType) { + case RESTATE: + assertEquals(Set.of(), exceptions, "Expected no exception for change type RESTATE"); + break; + case CREATE: + case CREATE_ENTITY: + assertEquals(exceptions.size(), 1, "Expected exception for change type " + changeType); + assertEquals( + exceptions.stream().findFirst().get().getMessage(), + "Expected version 2, actual version -1"); + break; + default: + assertEquals(exceptions.size(), 1, "Expected exception for change type " + changeType); + assertEquals( + exceptions.stream().findFirst().get().getMessage(), + "Expected version 2, actual version 3"); + break; + } + } + } + + @Test + public void testNoPreviousVersionsLookupSchemaMetadataNextVersionFail() { + ConditionalWriteValidator test = new ConditionalWriteValidator().setConfig(validatorConfig); + Urn testEntityUrn = UrnUtils.getUrn("urn:li:chart:(looker,baz1)"); + + // Prepare mock lookup based on version + when(mockRetrieverContext + .getAspectRetriever() + .getLatestSystemAspects(eq(Map.of(testEntityUrn, Set.of("status"))))) + .thenReturn( + Map.of( + testEntityUrn, + Map.of( + "status", + TestSystemAspect.builder() + .systemMetadata( + new SystemMetadata().setVersion("3")) // expected next version 3 + .build()))); + + for (ChangeType changeType : supportedChangeTypes) { + final ChangeMCP testMCP; + switch (changeType) { + case CREATE: + case CREATE_ENTITY: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected is always 1 + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "2")) + .build(); + break; + default: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "2")) + .build(); + break; + } + + Set exceptions = + test.validatePreCommit(List.of(testMCP), mockRetrieverContext) + .collect(Collectors.toSet()); + + switch (changeType) { + case RESTATE: + assertEquals(Set.of(), exceptions, "Expected no exception for change type RESTATE"); + break; + case CREATE: + case CREATE_ENTITY: + assertEquals(exceptions.size(), 1, "Expected exception for change type " + changeType); + assertEquals( + exceptions.stream().findFirst().get().getMessage(), + "Expected version 2, actual version -1"); + break; + default: + assertEquals(exceptions.size(), 1, "Expected exception for change type " + changeType); + assertEquals( + exceptions.stream().findFirst().get().getMessage(), + "Expected version 2, actual version 3"); + break; + } + } + } + + @Test + public void testNoPreviousVersionsLookupVersionNextVersionFail() { + ConditionalWriteValidator test = new ConditionalWriteValidator().setConfig(validatorConfig); + Urn testEntityUrn = UrnUtils.getUrn("urn:li:chart:(looker,baz1)"); + + // Prepare mock lookup based on version + when(mockRetrieverContext + .getAspectRetriever() + .getLatestSystemAspects(eq(Map.of(testEntityUrn, Set.of("status"))))) + .thenReturn( + Map.of( + testEntityUrn, + Map.of( + "status", + TestSystemAspect.builder() + .version(3) // expected next version 3 + .build()))); + + for (ChangeType changeType : supportedChangeTypes) { + final ChangeMCP testMCP; + switch (changeType) { + case CREATE: + case CREATE_ENTITY: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected is always 1 + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "2")) + .build(); + break; + default: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .headers(Map.of(HTTP_HEADER_IF_VERSION_MATCH, "2")) + .build(); + break; + } + + Set exceptions = + test.validatePreCommit(List.of(testMCP), mockRetrieverContext) + .collect(Collectors.toSet()); + + switch (changeType) { + case RESTATE: + assertEquals(Set.of(), exceptions, "Expected no exception for change type RESTATE"); + break; + case CREATE: + case CREATE_ENTITY: + assertEquals(exceptions.size(), 1, "Expected exception for change type " + changeType); + assertEquals( + exceptions.stream().findFirst().get().getMessage(), + "Expected version 2, actual version -1"); + break; + default: + assertEquals(exceptions.size(), 1, "Expected exception for change type " + changeType); + assertEquals( + exceptions.stream().findFirst().get().getMessage(), + "Expected version 2, actual version 3"); + break; + } + } + } + + @Test + public void testModifiedSinceSuccess() { + ConditionalWriteValidator test = new ConditionalWriteValidator().setConfig(validatorConfig); + Urn testEntityUrn = UrnUtils.getUrn("urn:li:chart:(looker,baz1)"); + + StringMap headers = + new StringMap( + Map.of( + HttpHeaders.IF_UNMODIFIED_SINCE, "2024-07-03T00:00:00Z", + HttpHeaders.IF_MODIFIED_SINCE, "2024-07-01T00:00:00Z")); + Timestamp modified = Timestamp.from(Instant.parse("2024-07-02T00:00:00Z")); + + for (ChangeType changeType : supportedChangeTypes) { + final ChangeMCP testMCP; + switch (changeType) { + case RESTATE: + case CREATE_ENTITY: + case CREATE: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .metadataChangeProposal( + new MetadataChangeProposal() + .setHeaders( + new StringMap( + Map.of( + HttpHeaders.IF_UNMODIFIED_SINCE, + headers.get(HttpHeaders.IF_UNMODIFIED_SINCE))))) + .build(); + break; + default: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .metadataChangeProposal(new MetadataChangeProposal().setHeaders(headers)) + .previousSystemAspect( + TestSystemAspect.builder() + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + // Previous / actual + .createdOn(modified) + .createdBy("urn:li:corpuser:test") + .build()) + .build(); + break; + } + + Set exceptions = + test.validatePreCommit(List.of(testMCP), mockRetrieverContext) + .collect(Collectors.toSet()); + + assertEquals(Set.of(), exceptions, "Expected no exceptions for change type " + changeType); + } + } + + @Test + public void testNoPreviousLookupAuditStampModifiedSinceSuccess() { + ConditionalWriteValidator test = new ConditionalWriteValidator().setConfig(validatorConfig); + Urn testEntityUrn = UrnUtils.getUrn("urn:li:chart:(looker,baz1)"); + + StringMap headers = + new StringMap( + Map.of( + HttpHeaders.IF_UNMODIFIED_SINCE, "2024-07-03T00:00:00Z", + HttpHeaders.IF_MODIFIED_SINCE, "2024-07-01T00:00:00Z")); + Timestamp modified = Timestamp.from(Instant.parse("2024-07-02T00:00:00Z")); + + // Prepare mock lookup + when(mockRetrieverContext + .getAspectRetriever() + .getLatestSystemAspects(eq(Map.of(testEntityUrn, Set.of("status"))))) + .thenReturn( + Map.of( + testEntityUrn, + Map.of( + "status", + TestSystemAspect.builder() + .createdOn(modified) + .createdBy("urn:li:corpuser:test") + .build()))); + + for (ChangeType changeType : supportedChangeTypes) { + final ChangeMCP testMCP; + switch (changeType) { + case CREATE: + case CREATE_ENTITY: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + .metadataChangeProposal( + new MetadataChangeProposal() + .setHeaders( + new StringMap( + Map.of( + HttpHeaders.IF_UNMODIFIED_SINCE, + headers.get(HttpHeaders.IF_UNMODIFIED_SINCE))))) + .build(); + break; + default: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .metadataChangeProposal(new MetadataChangeProposal().setHeaders(headers)) + .build(); + break; + } + + Set exceptions = + test.validatePreCommit(List.of(testMCP), mockRetrieverContext) + .collect(Collectors.toSet()); + + assertEquals(Set.of(), exceptions, "Expected no exceptions for change type " + changeType); + } + } + + @Test + public void testModifiedSinceBeforeRangeFail() { + ConditionalWriteValidator test = new ConditionalWriteValidator().setConfig(validatorConfig); + Urn testEntityUrn = UrnUtils.getUrn("urn:li:chart:(looker,baz1)"); + + StringMap headers = + new StringMap( + Map.of( + HttpHeaders.IF_UNMODIFIED_SINCE, "2024-07-03T00:00:00Z", + HttpHeaders.IF_MODIFIED_SINCE, "2024-07-01T00:00:00Z")); + Timestamp modified = Timestamp.from(Instant.parse("2024-06-30T00:00:00Z")); + + for (ChangeType changeType : supportedChangeTypes) { + final ChangeMCP testMCP; + switch (changeType) { + case RESTATE: + case CREATE_ENTITY: + case CREATE: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .metadataChangeProposal( + new MetadataChangeProposal() + .setHeaders( + new StringMap( + Map.of( + HttpHeaders.IF_UNMODIFIED_SINCE, + headers.get(HttpHeaders.IF_UNMODIFIED_SINCE))))) + .build(); + break; + default: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .metadataChangeProposal(new MetadataChangeProposal().setHeaders(headers)) + .previousSystemAspect( + TestSystemAspect.builder() + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + // Previous / actual + .createdOn(modified) + .createdBy("urn:li:corpuser:test") + .build()) + .build(); + break; + } + + Set exceptions = + test.validatePreCommit(List.of(testMCP), mockRetrieverContext) + .collect(Collectors.toSet()); + + switch (changeType) { + case CREATE: + case CREATE_ENTITY: + case RESTATE: + assertEquals(Set.of(), exceptions, "Expected no exception for change type " + changeType); + break; + default: + assertEquals(exceptions.size(), 1, "Expected exception for change type " + changeType); + assertEquals( + exceptions.stream().findFirst().get().getMessage(), + "Item last modified 1719705600000 <= 1719792000000 (epoch ms)"); + break; + } + } + } + + @Test + public void testModifiedSinceAfterRangeFail() { + ConditionalWriteValidator test = new ConditionalWriteValidator().setConfig(validatorConfig); + Urn testEntityUrn = UrnUtils.getUrn("urn:li:chart:(looker,baz1)"); + + StringMap headers = + new StringMap( + Map.of( + HttpHeaders.IF_UNMODIFIED_SINCE, "2024-07-03T00:00:00Z", + HttpHeaders.IF_MODIFIED_SINCE, "2024-07-01T00:00:00Z")); + Timestamp modified = Timestamp.from(Instant.parse("2024-07-04T00:00:00Z")); + + for (ChangeType changeType : supportedChangeTypes) { + final ChangeMCP testMCP; + switch (changeType) { + case RESTATE: + case CREATE_ENTITY: + case CREATE: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .metadataChangeProposal( + new MetadataChangeProposal() + .setHeaders( + new StringMap( + Map.of( + HttpHeaders.IF_UNMODIFIED_SINCE, + headers.get(HttpHeaders.IF_UNMODIFIED_SINCE))))) + .build(); + break; + default: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .metadataChangeProposal(new MetadataChangeProposal().setHeaders(headers)) + .previousSystemAspect( + TestSystemAspect.builder() + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + // Previous / actual + .createdOn(modified) + .createdBy("urn:li:corpuser:test") + .build()) + .build(); + break; + } + + Set exceptions = + test.validatePreCommit(List.of(testMCP), mockRetrieverContext) + .collect(Collectors.toSet()); + + switch (changeType) { + case CREATE: + case CREATE_ENTITY: + case RESTATE: + assertEquals(Set.of(), exceptions, "Expected no exception for change type " + changeType); + break; + default: + assertEquals(exceptions.size(), 1, "Expected exception for change type " + changeType); + assertEquals( + exceptions.stream().findFirst().get().getMessage(), + "Item last modified 1720051200000 > 1719964800000 (epoch ms)"); + break; + } + } + } + + @Test + public void testNoPreviousLookupAuditStampModifiedSinceBeforeRangeFail() { + ConditionalWriteValidator test = new ConditionalWriteValidator().setConfig(validatorConfig); + Urn testEntityUrn = UrnUtils.getUrn("urn:li:chart:(looker,baz1)"); + + StringMap headers = + new StringMap( + Map.of( + HttpHeaders.IF_UNMODIFIED_SINCE, "2024-07-03T00:00:00Z", + HttpHeaders.IF_MODIFIED_SINCE, "2024-07-01T00:00:00Z")); + Timestamp modified = Timestamp.from(Instant.parse("2024-06-30T00:00:00Z")); + + // Prepare mock lookup + when(mockRetrieverContext + .getAspectRetriever() + .getLatestSystemAspects(eq(Map.of(testEntityUrn, Set.of("status"))))) + .thenReturn( + Map.of( + testEntityUrn, + Map.of( + "status", + TestSystemAspect.builder() + .createdOn(modified) + .createdBy("urn:li:corpuser:test") + .build()))); + + for (ChangeType changeType : supportedChangeTypes) { + final ChangeMCP testMCP; + switch (changeType) { + case CREATE: + case CREATE_ENTITY: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + .metadataChangeProposal( + new MetadataChangeProposal() + .setHeaders( + new StringMap( + Map.of( + HttpHeaders.IF_UNMODIFIED_SINCE, + headers.get(HttpHeaders.IF_UNMODIFIED_SINCE))))) + .build(); + break; + default: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .metadataChangeProposal(new MetadataChangeProposal().setHeaders(headers)) + .build(); + break; + } + + Set exceptions = + test.validatePreCommit(List.of(testMCP), mockRetrieverContext) + .collect(Collectors.toSet()); + + switch (changeType) { + case CREATE: + case CREATE_ENTITY: + case RESTATE: + assertEquals(Set.of(), exceptions, "Expected no exception for change type " + changeType); + break; + default: + assertEquals(exceptions.size(), 1, "Expected exception for change type " + changeType); + assertEquals( + exceptions.stream().findFirst().get().getMessage(), + "Item last modified 1719705600000 <= 1719792000000 (epoch ms)"); + break; + } + } + } + + @Test + public void testNoPreviousLookupAuditStampModifiedSinceAfterRangeFail() { + ConditionalWriteValidator test = new ConditionalWriteValidator().setConfig(validatorConfig); + Urn testEntityUrn = UrnUtils.getUrn("urn:li:chart:(looker,baz1)"); + + StringMap headers = + new StringMap( + Map.of( + HttpHeaders.IF_UNMODIFIED_SINCE, "2024-07-03T00:00:00Z", + HttpHeaders.IF_MODIFIED_SINCE, "2024-07-01T00:00:00Z")); + Timestamp modified = Timestamp.from(Instant.parse("2024-07-04T00:00:00Z")); + + // Prepare mock lookup + when(mockRetrieverContext + .getAspectRetriever() + .getLatestSystemAspects(eq(Map.of(testEntityUrn, Set.of("status"))))) + .thenReturn( + Map.of( + testEntityUrn, + Map.of( + "status", + TestSystemAspect.builder() + .createdOn(modified) + .createdBy("urn:li:corpuser:test") + .build()))); + + for (ChangeType changeType : supportedChangeTypes) { + final ChangeMCP testMCP; + switch (changeType) { + case CREATE: + case CREATE_ENTITY: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + .metadataChangeProposal( + new MetadataChangeProposal() + .setHeaders( + new StringMap( + Map.of( + HttpHeaders.IF_UNMODIFIED_SINCE, + headers.get(HttpHeaders.IF_UNMODIFIED_SINCE))))) + .build(); + break; + default: + testMCP = + TestMCP.builder() + .changeType(changeType) + .urn(testEntityUrn) + .entitySpec(entityRegistry.getEntitySpec(testEntityUrn.getEntityType())) + .aspectSpec( + entityRegistry + .getEntitySpec(testEntityUrn.getEntityType()) + .getAspectSpec("status")) + .recordTemplate(new Status().setRemoved(false)) + // Expected + .metadataChangeProposal(new MetadataChangeProposal().setHeaders(headers)) + .build(); + break; + } + + Set exceptions = + test.validatePreCommit(List.of(testMCP), mockRetrieverContext) + .collect(Collectors.toSet()); + + switch (changeType) { + case CREATE: + case CREATE_ENTITY: + case RESTATE: + assertEquals(Set.of(), exceptions, "Expected no exception for change type " + changeType); + break; + default: + assertEquals(exceptions.size(), 1, "Expected exception for change type " + changeType); + assertEquals( + exceptions.stream().findFirst().get().getMessage(), + "Item last modified 1720051200000 > 1719964800000 (epoch ms)"); + break; + } + } + } +} diff --git a/entity-registry/src/test/java/com/linkedin/test/metadata/aspect/MockAspectRetriever.java b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/MockAspectRetriever.java similarity index 63% rename from entity-registry/src/test/java/com/linkedin/test/metadata/aspect/MockAspectRetriever.java rename to entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/MockAspectRetriever.java index 9c7cd997a9af0..65705f15022b6 100644 --- a/entity-registry/src/test/java/com/linkedin/test/metadata/aspect/MockAspectRetriever.java +++ b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/MockAspectRetriever.java @@ -1,25 +1,30 @@ package com.linkedin.test.metadata.aspect; -import static org.mockito.Mockito.mock; - import com.linkedin.common.Status; import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; import com.linkedin.entity.Aspect; import com.linkedin.metadata.aspect.AspectRetriever; +import com.linkedin.metadata.aspect.SystemAspect; import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.mxe.SystemMetadata; import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.test.metadata.aspect.batch.TestSystemAspect; import com.linkedin.util.Pair; +import java.sql.Timestamp; +import java.time.Instant; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nonnull; +import org.mockito.Mockito; public class MockAspectRetriever implements AspectRetriever { private final Map> data; + private final Map> systemData = new HashMap<>(); public MockAspectRetriever(@Nonnull Map> data) { this.data = @@ -40,6 +45,21 @@ public MockAspectRetriever(@Nonnull Map> data) { }) .collect(Collectors.toMap(Pair::getKey, Pair::getValue)))) .collect(Collectors.toMap(Pair::getKey, Pair::getValue))); + for (Map.Entry> urnEntry : this.data.entrySet()) { + for (Map.Entry aspectEntry : urnEntry.getValue().entrySet()) { + systemData + .computeIfAbsent(urnEntry.getKey(), urn -> new HashMap<>()) + .computeIfAbsent( + aspectEntry.getKey(), + aspectName -> + TestSystemAspect.builder() + .urn(urnEntry.getKey()) + .version(0) + .systemMetadata(new SystemMetadata().setVersion("1")) + .createdOn(Timestamp.from(Instant.now())) + .build()); + } + } } public MockAspectRetriever( @@ -61,9 +81,19 @@ public Map> getLatestAspectObjects( .collect(Collectors.toMap(Pair::getKey, Pair::getValue)); } + @Nonnull + @Override + public Map> getLatestSystemAspects( + Map> urnAspectNames) { + return urnAspectNames.keySet().stream() + .filter(systemData::containsKey) + .map(urn -> Pair.of(urn, systemData.get(urn))) + .collect(Collectors.toMap(Pair::getKey, Pair::getValue)); + } + @Nonnull @Override public EntityRegistry getEntityRegistry() { - return mock(EntityRegistry.class); + return Mockito.mock(EntityRegistry.class); } } diff --git a/entity-registry/src/test/java/com/linkedin/test/metadata/aspect/TestEntityRegistry.java b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/TestEntityRegistry.java similarity index 100% rename from entity-registry/src/test/java/com/linkedin/test/metadata/aspect/TestEntityRegistry.java rename to entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/TestEntityRegistry.java diff --git a/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCL.java b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCL.java new file mode 100644 index 0000000000000..7dd889c48b874 --- /dev/null +++ b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCL.java @@ -0,0 +1,32 @@ +package com.linkedin.test.metadata.aspect.batch; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.batch.MCLItem; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.mxe.MetadataChangeLog; +import javax.annotation.Nonnull; +import lombok.Builder; +import lombok.Getter; + +@Builder(toBuilder = true) +@Getter +public class TestMCL implements MCLItem { + private Urn urn; + private ChangeType changeType; + private MetadataChangeLog metadataChangeLog; + private RecordTemplate previousRecordTemplate; + private RecordTemplate recordTemplate; + private EntitySpec entitySpec; + private AspectSpec aspectSpec; + private AuditStamp auditStamp; + + @Nonnull + @Override + public String getAspectName() { + return getAspectSpec().getName(); + } +} diff --git a/entity-registry/src/test/java/com/linkedin/test/metadata/aspect/batch/TestMCP.java b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCP.java similarity index 90% rename from entity-registry/src/test/java/com/linkedin/test/metadata/aspect/batch/TestMCP.java rename to entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCP.java index 1e1efe4238187..e562390a959a7 100644 --- a/entity-registry/src/test/java/com/linkedin/test/metadata/aspect/batch/TestMCP.java +++ b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCP.java @@ -20,7 +20,10 @@ import com.linkedin.test.metadata.aspect.TestEntityRegistry; import java.net.URISyntaxException; import java.util.Collection; +import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.Builder; @@ -119,10 +122,22 @@ public static Set ofOneMCP( private MetadataChangeProposal metadataChangeProposal; @Setter private SystemAspect previousSystemAspect; @Setter private long nextAspectVersion; + @Setter private Map headers; @Nonnull @Override public SystemAspect getSystemAspect(@Nullable Long nextAspectVersion) { return null; } + + @Override + public Map getHeaders() { + return Optional.ofNullable(metadataChangeProposal) + .filter(MetadataChangeProposal::hasHeaders) + .map( + mcp -> + mcp.getHeaders().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) + .orElse(headers); + } } diff --git a/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestSystemAspect.java b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestSystemAspect.java new file mode 100644 index 0000000000000..54afdd214f2b2 --- /dev/null +++ b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestSystemAspect.java @@ -0,0 +1,26 @@ +package com.linkedin.test.metadata.aspect.batch; + +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.aspect.SystemAspect; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.mxe.SystemMetadata; +import java.sql.Timestamp; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +@Builder +@Getter +@EqualsAndHashCode +public class TestSystemAspect implements SystemAspect { + private Urn urn; + private long version; + private RecordTemplate recordTemplate; + private SystemMetadata systemMetadata; + private EntitySpec entitySpec; + private AspectSpec aspectSpec; + private Timestamp createdOn; + private String createdBy; +} diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index 66ed48a428a21..9a7b8287e2c6a 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -21,6 +21,10 @@ public class Constants { public static final String STRUCTURED_PROPERTY_MAPPING_FIELD = "structuredProperties"; public static final String STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX = STRUCTURED_PROPERTY_MAPPING_FIELD + "."; + public static final String STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD = "_versioned"; + public static final String STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD_PREFIX = + String.join( + ".", STRUCTURED_PROPERTY_MAPPING_FIELD, STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD, ""); // !!!!!!! IMPORTANT !!!!!!! // This effectively sets the max aspect size to 16 MB. Used in deserialization of messages. @@ -285,6 +289,7 @@ public class Constants { public static final String ASSERTION_INFO_ASPECT_NAME = "assertionInfo"; public static final String ASSERTION_RUN_EVENT_ASPECT_NAME = "assertionRunEvent"; public static final String ASSERTION_RUN_EVENT_STATUS_COMPLETE = "COMPLETE"; + public static final String ASSERTION_ACTIONS_ASPECT_NAME = "assertionActions"; // Tests public static final String TEST_ENTITY_NAME = "test"; @@ -341,6 +346,7 @@ public class Constants { // Structured Property public static final String STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME = "propertyDefinition"; + public static final String STRUCTURED_PROPERTY_KEY_ASPECT_NAME = "structuredPropertyKey"; // Form public static final String FORM_INFO_ASPECT_NAME = "formInfo"; @@ -362,6 +368,12 @@ public class Constants { public static final String DATAHUB_CONNECTION_ENTITY_NAME = "dataHubConnection"; public static final String DATAHUB_CONNECTION_DETAILS_ASPECT_NAME = "dataHubConnectionDetails"; + // Data Contracts + public static final String DATA_CONTRACT_ENTITY_NAME = "dataContract"; + public static final String DATA_CONTRACT_PROPERTIES_ASPECT_NAME = "dataContractProperties"; + public static final String DATA_CONTRACT_KEY_ASPECT_NAME = "dataContractKey"; + public static final String DATA_CONTRACT_STATUS_ASPECT_NAME = "dataContractStatus"; + // Relationships public static final String IS_MEMBER_OF_GROUP_RELATIONSHIP_NAME = "IsMemberOfGroup"; public static final String IS_MEMBER_OF_NATIVE_GROUP_RELATIONSHIP_NAME = "IsMemberOfNativeGroup"; diff --git a/metadata-dao-impl/kafka-producer/build.gradle b/metadata-dao-impl/kafka-producer/build.gradle index bc3415b2ccc8c..2df15309810db 100644 --- a/metadata-dao-impl/kafka-producer/build.gradle +++ b/metadata-dao-impl/kafka-producer/build.gradle @@ -18,6 +18,7 @@ dependencies { annotationProcessor externalDependency.lombok testImplementation externalDependency.mockito + testImplementation externalDependency.testng constraints { implementation(externalDependency.log4jCore) { diff --git a/metadata-dao-impl/kafka-producer/src/main/java/com/datahub/metadata/dao/producer/KafkaProducerThrottle.java b/metadata-dao-impl/kafka-producer/src/main/java/com/datahub/metadata/dao/producer/KafkaProducerThrottle.java new file mode 100644 index 0000000000000..8fbb34b1eacd6 --- /dev/null +++ b/metadata-dao-impl/kafka-producer/src/main/java/com/datahub/metadata/dao/producer/KafkaProducerThrottle.java @@ -0,0 +1,246 @@ +package com.datahub.metadata.dao.producer; + +import com.codahale.metrics.Gauge; +import com.google.common.annotations.VisibleForTesting; +import com.linkedin.metadata.config.MetadataChangeProposalConfig; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.utils.metrics.MetricUtils; +import com.linkedin.util.Pair; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.OffsetSpec; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; +import org.springframework.util.backoff.BackOffExecution; +import org.springframework.util.backoff.ExponentialBackOff; + +@Slf4j +@Builder(toBuilder = true) +public class KafkaProducerThrottle { + @Nonnull private final EntityRegistry entityRegistry; + @Nonnull private final Admin kafkaAdmin; + @Nonnull private final MetadataChangeProposalConfig.ThrottlesConfig config; + @Nonnull private final String mclConsumerGroupId; + @Nonnull private final String versionedTopicName; + @Nonnull private final String timeseriesTopicName; + @Nonnull private final Consumer pauseConsumer; + + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + private final Map medianLag = new ConcurrentHashMap<>(); + private final Map backoffMap = new ConcurrentHashMap<>(); + + /** Update lag information at a given rate */ + public KafkaProducerThrottle start() { + if ((config.getVersioned().isEnabled() || config.getTimeseries().isEnabled()) + && config.getUpdateIntervalMs() > 0) { + scheduler.scheduleAtFixedRate( + () -> { + refresh(); + try { + throttle(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }, + config.getUpdateIntervalMs(), + config.getUpdateIntervalMs(), + TimeUnit.MILLISECONDS); + } + return this; + } + + @VisibleForTesting + public void refresh() { + medianLag.putAll(getMedianLag()); + log.info("MCL medianLag: {}", medianLag); + } + + @VisibleForTesting + public void stop() { + scheduler.shutdown(); + } + + /** + * Get copy of the lag info + * + * @return median lag per mcl topic + */ + @VisibleForTesting + public Map getLag() { + return medianLag.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + @VisibleForTesting + public boolean isThrottled(MclType mclType) { + if (getThrottleConfig(mclType).isEnabled() && medianLag.containsKey(mclType)) { + return medianLag.get(mclType) > getThrottleConfig(mclType).getThreshold(); + } + return false; + } + + @VisibleForTesting + public long computeNextBackOff(MclType mclType) { + if (isThrottled(mclType)) { + BackOffExecution backOffExecution = + backoffMap.computeIfAbsent( + mclType, + k -> { + MetadataChangeProposalConfig.ThrottleConfig throttleConfig = + getThrottleConfig(mclType); + ExponentialBackOff backoff = + new ExponentialBackOff( + throttleConfig.getInitialIntervalMs(), throttleConfig.getMultiplier()); + backoff.setMaxAttempts(throttleConfig.getMaxAttempts()); + backoff.setMaxInterval(throttleConfig.getMaxIntervalMs()); + return backoff.start(); + }); + return backOffExecution.nextBackOff(); + } + return 0; + } + + @VisibleForTesting + public void throttle() throws InterruptedException { + for (MclType mclType : MclType.values()) { + if (isThrottled(mclType)) { + long backoffWaitMs = computeNextBackOff(mclType); + + if (backoffWaitMs > 0) { + log.warn( + "Throttled producer Topic: {} Duration: {} ms MedianLag: {}", + getTopicName(mclType), + backoffWaitMs, + medianLag.get(mclType)); + MetricUtils.gauge( + this.getClass(), + String.format("%s_throttled", getTopicName(mclType)), + () -> (Gauge) () -> 1); + MetricUtils.counter( + this.getClass(), String.format("%s_throttledCount", getTopicName(mclType))) + .inc(); + + log.info("Pausing MCE consumer for {} ms.", backoffWaitMs); + pauseConsumer.accept(true); + Thread.sleep(backoffWaitMs); + log.info("Resuming MCE consumer."); + pauseConsumer.accept(false); + + // if throttled for one topic, skip remaining + return; + } else { + // no throttle or exceeded configuration limits + log.info("MCE consumer throttle exponential backoff reset."); + backoffMap.remove(mclType); + MetricUtils.gauge( + this.getClass(), + String.format("%s_throttled", getTopicName(mclType)), + () -> (Gauge) () -> 0); + } + } else { + // not throttled, remove backoff tracking + log.info("MCE consumer throttle exponential backoff reset."); + backoffMap.remove(mclType); + MetricUtils.gauge( + this.getClass(), + String.format("%s_throttled", getTopicName(mclType)), + () -> (Gauge) () -> 0); + } + } + } + + private Map getMedianLag() { + try { + Map mclConsumerOffsets = + kafkaAdmin + .listConsumerGroupOffsets(mclConsumerGroupId) + .partitionsToOffsetAndMetadata() + .get() + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + Map latestOffsetRequest = + mclConsumerOffsets.keySet().stream() + .map(offsetAndMetadata -> Map.entry(offsetAndMetadata, OffsetSpec.latest())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map endOffsetValues = + kafkaAdmin.listOffsets(latestOffsetRequest).all().get().entrySet().stream() + .map(entry -> Map.entry(entry.getKey(), entry.getValue().offset())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + return Stream.of( + Pair.of(MclType.VERSIONED, versionedTopicName), + Pair.of(MclType.TIMESERIES, timeseriesTopicName)) + .map( + topic -> { + MclType mclType = topic.getFirst(); + String topicName = topic.getSecond(); + + Map topicOffsets = + mclConsumerOffsets.entrySet().stream() + .filter(entry -> entry.getKey().topic().equals(topicName)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + List offsetValues = + topicOffsets.values().stream() + .map(OffsetAndMetadata::offset) + .map(Long::doubleValue) + .collect(Collectors.toList()); + long offsetMedian = getMedian(offsetValues).longValue(); + + List topicEndOffsetValues = + topicOffsets.keySet().stream() + .map(topicPart -> endOffsetValues.getOrDefault(topicPart, 0L)) + .map(Long::doubleValue) + .collect(Collectors.toList()); + long endOffsetMedian = getMedian(topicEndOffsetValues).longValue(); + return Map.entry(mclType, Math.max(0, endOffsetMedian - offsetMedian)); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } catch (ExecutionException | InterruptedException e) { + log.error("Error fetching consumer group offsets.", e); + return Map.of(MclType.VERSIONED, 0L, MclType.TIMESERIES, 0L); + } + } + + private MetadataChangeProposalConfig.ThrottleConfig getThrottleConfig(MclType mclType) { + MetadataChangeProposalConfig.ThrottleConfig throttleConfig; + switch (mclType) { + case VERSIONED -> throttleConfig = config.getVersioned(); + case TIMESERIES -> throttleConfig = config.getTimeseries(); + default -> throw new IllegalStateException(); + } + return throttleConfig; + } + + private String getTopicName(MclType mclType) { + return MclType.TIMESERIES.equals(mclType) ? timeseriesTopicName : versionedTopicName; + } + + private static Double getMedian(Collection listValues) { + double[] values = listValues.stream().mapToDouble(d -> d).sorted().toArray(); + double median; + if (values.length % 2 == 0) + median = (values[values.length / 2] + values[values.length / 2 - 1]) / 2; + else median = values[values.length / 2]; + return median; + } + + public enum MclType { + TIMESERIES, + VERSIONED + } +} diff --git a/metadata-dao-impl/kafka-producer/src/test/java/com/datahub/metadata/dao/producer/KafkaProducerThrottleTest.java b/metadata-dao-impl/kafka-producer/src/test/java/com/datahub/metadata/dao/producer/KafkaProducerThrottleTest.java new file mode 100644 index 0000000000000..ce6104ee2ca7d --- /dev/null +++ b/metadata-dao-impl/kafka-producer/src/test/java/com/datahub/metadata/dao/producer/KafkaProducerThrottleTest.java @@ -0,0 +1,363 @@ +package com.datahub.metadata.dao.producer; + +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +import com.linkedin.metadata.config.MetadataChangeProposalConfig; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.mxe.Topics; +import com.linkedin.util.Pair; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsResult; +import org.apache.kafka.clients.admin.ListOffsetsResult; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.TopicPartition; +import org.testng.annotations.Test; + +public class KafkaProducerThrottleTest { + private static final List STANDARD_TOPICS = + List.of(Topics.METADATA_CHANGE_LOG_VERSIONED, Topics.METADATA_CHANGE_LOG_TIMESERIES); + private static final String STANDARD_MCL_CONSUMER_GROUP_ID = "generic-mae-consumer-job-client"; + + @Test + public void testLagCalculation() throws ExecutionException, InterruptedException { + // 3 partitions + // Consumer offsets: 1, 2, 3 + // End offsets: 2, 4, 6 + // Lag: 1, 2, 3 + // MedianLag: 2 + AdminClient mockAdmin = + mockKafka( + generateLag( + STANDARD_TOPICS, + topicPart -> (long) topicPart.partition() + 1, + topicPart -> ((long) topicPart.partition() + 1) * 2, + 3)); + + KafkaProducerThrottle test = + KafkaProducerThrottle.builder() + .config(noSchedulerConfig().getThrottle()) + .kafkaAdmin(mockAdmin) + .versionedTopicName(STANDARD_TOPICS.get(0)) + .timeseriesTopicName(STANDARD_TOPICS.get(1)) + .entityRegistry(mock(EntityRegistry.class)) + .mclConsumerGroupId(STANDARD_MCL_CONSUMER_GROUP_ID) + .pauseConsumer(mock(Consumer.class)) + .build(); + + // Refresh calculations + test.refresh(); + + assertEquals( + test.getLag(), + Map.of( + KafkaProducerThrottle.MclType.VERSIONED, 2L, + KafkaProducerThrottle.MclType.TIMESERIES, 2L)); + } + + @Test + public void testThrottle() throws ExecutionException, InterruptedException { + MetadataChangeProposalConfig.ThrottlesConfig noThrottleConfig = + noSchedulerConfig().getThrottle(); + noThrottleConfig + .getVersioned() + .setThreshold(10) + .setInitialIntervalMs(1) + .setMultiplier(1) + .setMaxAttempts(1) + .setMaxIntervalMs(1); + + MetadataChangeProposalConfig.ThrottlesConfig throttleConfig = noSchedulerConfig().getThrottle(); + throttleConfig + .getVersioned() + .setThreshold(1) + .setInitialIntervalMs(1) + .setMultiplier(1) + .setMaxAttempts(1) + .setMaxIntervalMs(1); + + // 3 partitions + // Consumer offsets: 1, 2, 3 + // End offsets: 2, 4, 6 + // Lag: 1, 2, 3 + // MedianLag: 2 + AdminClient mockAdmin = + mockKafka( + generateLag( + STANDARD_TOPICS, + topicPart -> (long) topicPart.partition() + 1, + topicPart -> ((long) topicPart.partition() + 1) * 2, + 3)); + + Consumer pauseFunction = mock(Consumer.class); + + KafkaProducerThrottle test = + KafkaProducerThrottle.builder() + .config(noThrottleConfig) + .kafkaAdmin(mockAdmin) + .versionedTopicName(STANDARD_TOPICS.get(0)) + .timeseriesTopicName(STANDARD_TOPICS.get(1)) + .entityRegistry(mock(EntityRegistry.class)) + .mclConsumerGroupId(STANDARD_MCL_CONSUMER_GROUP_ID) + .pauseConsumer(pauseFunction) + .build(); + + // Refresh calculations + test.refresh(); + assertEquals( + test.getLag(), + Map.of( + KafkaProducerThrottle.MclType.VERSIONED, 2L, + KafkaProducerThrottle.MclType.TIMESERIES, 2L)); + assertFalse( + test.isThrottled(KafkaProducerThrottle.MclType.VERSIONED), + "Expected not throttling, lag is below threshold"); + assertFalse(test.isThrottled(KafkaProducerThrottle.MclType.TIMESERIES)); + test.throttle(); + verifyNoInteractions(pauseFunction); + reset(pauseFunction); + + KafkaProducerThrottle test2 = test.toBuilder().config(throttleConfig).build(); + // Refresh calculations + test2.refresh(); + assertEquals( + test2.getLag(), + Map.of( + KafkaProducerThrottle.MclType.VERSIONED, 2L, + KafkaProducerThrottle.MclType.TIMESERIES, 2L)); + assertTrue( + test2.isThrottled(KafkaProducerThrottle.MclType.VERSIONED), + "Expected throttling, lag is above threshold."); + assertFalse( + test2.isThrottled(KafkaProducerThrottle.MclType.TIMESERIES), + "Expected not throttling. Timeseries is disabled"); + test2.throttle(); + + // verify 1ms pause and resume + verify(pauseFunction).accept(eq(true)); + verify(pauseFunction).accept(eq(false)); + verifyNoMoreInteractions(pauseFunction); + } + + @Test + public void testBackOff() throws ExecutionException, InterruptedException { + MetadataChangeProposalConfig.ThrottlesConfig throttleConfig = noSchedulerConfig().getThrottle(); + throttleConfig + .getVersioned() + .setThreshold(1) + .setInitialIntervalMs(1) + .setMultiplier(2) + .setMaxAttempts(5) + .setMaxIntervalMs(8); + + // 3 partitions + // Consumer offsets: 1, 2, 3 + // End offsets: 2, 4, 6 + // Lag: 1, 2, 3 + // MedianLag: 2 + AdminClient mockAdmin = + mockKafka( + generateLag( + STANDARD_TOPICS, + topicPart -> (long) topicPart.partition() + 1, + topicPart -> ((long) topicPart.partition() + 1) * 2, + 3)); + + KafkaProducerThrottle test = + KafkaProducerThrottle.builder() + .config(throttleConfig) + .kafkaAdmin(mockAdmin) + .versionedTopicName(STANDARD_TOPICS.get(0)) + .timeseriesTopicName(STANDARD_TOPICS.get(1)) + .entityRegistry(mock(EntityRegistry.class)) + .mclConsumerGroupId(STANDARD_MCL_CONSUMER_GROUP_ID) + .pauseConsumer(mock(Consumer.class)) + .build(); + + // Refresh calculations + test.refresh(); + assertEquals( + test.getLag(), + Map.of( + KafkaProducerThrottle.MclType.VERSIONED, 2L, + KafkaProducerThrottle.MclType.TIMESERIES, 2L)); + assertTrue( + test.isThrottled(KafkaProducerThrottle.MclType.VERSIONED), + "Expected throttling, lag is above threshold."); + assertFalse( + test.isThrottled(KafkaProducerThrottle.MclType.TIMESERIES), + "Expected no throttling. Timeseries is disabled"); + + assertEquals( + test.computeNextBackOff(KafkaProducerThrottle.MclType.TIMESERIES), + 0L, + "Expected no backoff. Timeseries is disabled."); + + assertEquals( + test.computeNextBackOff(KafkaProducerThrottle.MclType.VERSIONED), 1L, "Expected initial 1"); + assertEquals( + test.computeNextBackOff(KafkaProducerThrottle.MclType.VERSIONED), + 2L, + "Expected second 2^1"); + assertEquals( + test.computeNextBackOff(KafkaProducerThrottle.MclType.VERSIONED), 4L, "Expected third 2^2"); + assertEquals( + test.computeNextBackOff(KafkaProducerThrottle.MclType.VERSIONED), + 8L, + "Expected fourth 2^3"); + assertEquals( + test.computeNextBackOff(KafkaProducerThrottle.MclType.VERSIONED), + 8L, + "Expected fifth max interval at 8"); + assertEquals( + test.computeNextBackOff(KafkaProducerThrottle.MclType.VERSIONED), + -1L, + "Expected max attempts"); + } + + @Test + public void testScheduler() throws ExecutionException, InterruptedException { + MetadataChangeProposalConfig config = new MetadataChangeProposalConfig(); + MetadataChangeProposalConfig.ThrottlesConfig throttlesConfig = + new MetadataChangeProposalConfig.ThrottlesConfig() + .setUpdateIntervalMs(10); // configure fast update for test + throttlesConfig.setVersioned( + new MetadataChangeProposalConfig.ThrottleConfig() + .setEnabled(true) // enable 1 throttle config to activate + ); + throttlesConfig.setTimeseries( + new MetadataChangeProposalConfig.ThrottleConfig().setEnabled(false)); + config.setThrottle(throttlesConfig); + + // 1 lag, 1 partition + AdminClient mockAdmin = + mockKafka(generateLag(STANDARD_TOPICS, topicPart -> 1L, topicPart -> 2L, 1)); + + KafkaProducerThrottle test = + KafkaProducerThrottle.builder() + .config(throttlesConfig) + .kafkaAdmin(mockAdmin) + .versionedTopicName(STANDARD_TOPICS.get(0)) + .timeseriesTopicName(STANDARD_TOPICS.get(1)) + .entityRegistry(mock(EntityRegistry.class)) + .mclConsumerGroupId(STANDARD_MCL_CONSUMER_GROUP_ID) + .pauseConsumer(mock(Consumer.class)) + .build(); + + try { + test.start(); + Thread.sleep(50); + assertEquals( + test.getLag(), + Map.of( + KafkaProducerThrottle.MclType.VERSIONED, 1L, + KafkaProducerThrottle.MclType.TIMESERIES, 1L), + "Expected lag updated"); + } finally { + test.stop(); + } + } + + private static MetadataChangeProposalConfig noSchedulerConfig() { + MetadataChangeProposalConfig config = new MetadataChangeProposalConfig(); + MetadataChangeProposalConfig.ThrottlesConfig throttlesConfig = + new MetadataChangeProposalConfig.ThrottlesConfig() + .setUpdateIntervalMs(0); // no scheduler, manual update + throttlesConfig.setVersioned( + new MetadataChangeProposalConfig.ThrottleConfig() + .setEnabled(true) // enable 1 throttle config to activate + ); + throttlesConfig.setTimeseries( + new MetadataChangeProposalConfig.ThrottleConfig().setEnabled(false)); + config.setThrottle(throttlesConfig); + return config; + } + + private static Pair, Map> + generateLag( + Collection topicNames, + Function consumerOffset, + Function endOffset, + int partitions) { + + Set topicPartitions = + topicNames.stream() + .flatMap( + topicName -> + IntStream.range(0, partitions) + .mapToObj(partitionNum -> new TopicPartition(topicName, partitionNum))) + .collect(Collectors.toSet()); + + Map consumerOffsetMap = + topicPartitions.stream() + .map( + topicPartition -> + Map.entry( + topicPartition, + new OffsetAndMetadata(consumerOffset.apply(topicPartition)))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map endOffsetMap = + topicPartitions.stream() + .map(topicPartition -> Map.entry(topicPartition, endOffset.apply(topicPartition))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + return Pair.of(consumerOffsetMap, endOffsetMap); + } + + private static AdminClient mockKafka( + Pair, Map> offsetPair) + throws ExecutionException, InterruptedException { + + AdminClient mockKafkaAdmin = mock(AdminClient.class); + + // consumer offsets + ListConsumerGroupOffsetsResult mockConsumerOffsetsResult = + mock(ListConsumerGroupOffsetsResult.class); + KafkaFuture> mockConsumerFuture = + mock(KafkaFuture.class); + when(mockConsumerOffsetsResult.partitionsToOffsetAndMetadata()).thenReturn(mockConsumerFuture); + when(mockConsumerFuture.get()).thenReturn(offsetPair.getFirst()); + when(mockKafkaAdmin.listConsumerGroupOffsets(anyString())) + .thenReturn(mockConsumerOffsetsResult); + + // end offsets + ListOffsetsResult mockOffsetsResult = mock(ListOffsetsResult.class); + KafkaFuture> mockOffsetFuture = + mock(KafkaFuture.class); + Map resultMap = + offsetPair.getSecond().entrySet().stream() + .map( + entry -> { + ListOffsetsResult.ListOffsetsResultInfo mockInfo = + mock(ListOffsetsResult.ListOffsetsResultInfo.class); + when(mockInfo.offset()).thenReturn(entry.getValue()); + return Map.entry(entry.getKey(), mockInfo); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + when(mockOffsetFuture.get()).thenReturn(resultMap); + when(mockOffsetsResult.all()).thenReturn(mockOffsetFuture); + when(mockKafkaAdmin.listOffsets(anyMap())).thenReturn(mockOffsetsResult); + + return mockKafkaAdmin; + } +} diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py index f2cd647837d5d..c37a1b334ed37 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py @@ -28,9 +28,12 @@ class DatahubLineageConfig(ConfigModel): # Cluster to associate with the pipelines and tasks. Defaults to "prod". cluster: str = builder.DEFAULT_FLOW_CLUSTER - # If true, the owners field of the DAG will be capture as a DataHub corpuser. + # If true, the owners field of the DAG will be captured as a DataHub corpuser. capture_ownership_info: bool = True + # If true, the owners field of the DAG will instead be captured as a DataHub corpgroup. + capture_ownership_as_group: bool = False + # If true, the tags field of the DAG will be captured as DataHub tags. capture_tags_info: bool = True @@ -70,6 +73,9 @@ def get_lineage_config() -> DatahubLineageConfig: capture_ownership_info = conf.get( "datahub", "capture_ownership_info", fallback=True ) + capture_ownership_as_group = conf.get( + "datahub", "capture_ownership_as_group", fallback=False + ) capture_executions = conf.get("datahub", "capture_executions", fallback=True) materialize_iolets = conf.get("datahub", "materialize_iolets", fallback=True) enable_extractors = conf.get("datahub", "enable_extractors", fallback=True) @@ -87,6 +93,7 @@ def get_lineage_config() -> DatahubLineageConfig: datahub_conn_id=datahub_conn_id, cluster=cluster, capture_ownership_info=capture_ownership_info, + capture_ownership_as_group=capture_ownership_as_group, capture_tags_info=capture_tags_info, capture_executions=capture_executions, materialize_iolets=materialize_iolets, diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py index d67754605c71b..8aa154dc267b6 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/client/airflow_generator.py @@ -129,10 +129,8 @@ def _get_dependencies( @staticmethod def generate_dataflow( - cluster: str, + config: DatahubLineageConfig, dag: "DAG", - capture_owner: bool = True, - capture_tags: bool = True, ) -> DataFlow: """ Generates a Dataflow object from an Airflow DAG @@ -146,7 +144,10 @@ def generate_dataflow( orchestrator = "airflow" description = "\n\n".join(filter(None, [dag.description, dag.doc_md])) or None data_flow = DataFlow( - env=cluster, id=id, orchestrator=orchestrator, description=description + env=config.cluster, + id=id, + orchestrator=orchestrator, + description=description, ) flow_property_bag: Dict[str, str] = {} @@ -173,10 +174,14 @@ def generate_dataflow( base_url = conf.get("webserver", "base_url") data_flow.url = f"{base_url}/tree?dag_id={dag.dag_id}" - if capture_owner and dag.owner: - data_flow.owners.update(owner.strip() for owner in dag.owner.split(",")) + if config.capture_ownership_info and dag.owner: + owners = [owner.strip() for owner in dag.owner.split(",")] + if config.capture_ownership_as_group: + data_flow.group_owners.update(owners) + else: + data_flow.owners.update(owners) - if capture_tags and dag.tags: + if config.capture_tags_info and dag.tags: data_flow.tags.update(dag.tags) return data_flow @@ -277,7 +282,10 @@ def generate_datajob( datajob.url = f"{base_url}/taskinstance/list/?flt1_dag_id_equals={datajob.flow_urn.flow_id}&_flt_3_task_id={task.task_id}" if capture_owner and dag.owner: - datajob.owners.add(dag.owner) + if config and config.capture_ownership_as_group: + datajob.group_owners.add(dag.owner) + else: + datajob.owners.add(dag.owner) if capture_tags and dag.tags: datajob.tags.update(dag.tags) @@ -311,14 +319,14 @@ def create_datajob_instance( @staticmethod def run_dataflow( emitter: Emitter, - cluster: str, + config: DatahubLineageConfig, dag_run: "DagRun", start_timestamp_millis: Optional[int] = None, dataflow: Optional[DataFlow] = None, ) -> None: if dataflow is None: assert dag_run.dag - dataflow = AirflowGenerator.generate_dataflow(cluster, dag_run.dag) + dataflow = AirflowGenerator.generate_dataflow(config, dag_run.dag) if start_timestamp_millis is None: assert dag_run.execution_date @@ -357,13 +365,15 @@ def run_dataflow( dpi.properties.update(property_bag) dpi.emit_process_start( - emitter=emitter, start_timestamp_millis=start_timestamp_millis + emitter=emitter, + start_timestamp_millis=start_timestamp_millis, + materialize_iolets=config.materialize_iolets, ) @staticmethod def complete_dataflow( emitter: Emitter, - cluster: str, + config: DatahubLineageConfig, dag_run: "DagRun", end_timestamp_millis: Optional[int] = None, dataflow: Optional[DataFlow] = None, @@ -378,7 +388,7 @@ def complete_dataflow( """ if dataflow is None: assert dag_run.dag - dataflow = AirflowGenerator.generate_dataflow(cluster, dag_run.dag) + dataflow = AirflowGenerator.generate_dataflow(config, dag_run.dag) assert dag_run.run_id dpi = DataProcessInstance.from_dataflow(dataflow=dataflow, id=dag_run.run_id) @@ -409,28 +419,27 @@ def complete_dataflow( @staticmethod def run_datajob( emitter: Emitter, - cluster: str, ti: "TaskInstance", dag: "DAG", dag_run: "DagRun", + config: DatahubLineageConfig, start_timestamp_millis: Optional[int] = None, datajob: Optional[DataJob] = None, attempt: Optional[int] = None, emit_templates: bool = True, - config: Optional[DatahubLineageConfig] = None, ) -> DataProcessInstance: if datajob is None: assert ti.task is not None datajob = AirflowGenerator.generate_datajob( - cluster, ti.task, dag, config=config + config.cluster, ti.task, dag, config=config ) assert dag_run.run_id dpi = DataProcessInstance.from_datajob( datajob=datajob, id=f"{dag.dag_id}_{ti.task_id}_{dag_run.run_id}", - clone_inlets=config is None or config.materialize_iolets, - clone_outlets=config is None or config.materialize_iolets, + clone_inlets=True, + clone_outlets=True, ) job_property_bag: Dict[str, str] = {} job_property_bag["run_id"] = str(dag_run.run_id) @@ -481,6 +490,7 @@ def run_datajob( start_timestamp_millis=start_timestamp_millis, attempt=attempt, emit_template=emit_templates, + materialize_iolets=config.materialize_iolets, ) return dpi diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py index ed155a35a925c..6ef4f831522cb 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py @@ -1,3 +1,4 @@ +import asyncio import copy import functools import logging @@ -8,12 +9,17 @@ import airflow import datahub.emitter.mce_builder as builder +from airflow.models.serialized_dag import SerializedDagModel from datahub.api.entities.datajob import DataJob from datahub.api.entities.dataprocess.dataprocess_instance import InstanceRunResult from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.rest_emitter import DatahubRestEmitter from datahub.ingestion.graph.client import DataHubGraph from datahub.metadata.schema_classes import ( + BrowsePathEntryClass, + BrowsePathsV2Class, + DataFlowKeyClass, + DataJobKeyClass, FineGrainedLineageClass, FineGrainedLineageDownstreamTypeClass, FineGrainedLineageUpstreamTypeClass, @@ -68,6 +74,7 @@ def hookimpl(f: _F) -> _F: # type: ignore[misc] # noqa: F811 "1", ) _RUN_IN_THREAD_TIMEOUT = 30 +_DATAHUB_CLEANUP_DAG = "Datahub_Cleanup" def get_airflow_plugin_listener() -> Optional["DataHubListener"]: @@ -403,13 +410,12 @@ def on_task_instance_running( if self.config.capture_executions: dpi = AirflowGenerator.run_datajob( emitter=self.emitter, - cluster=self.config.cluster, + config=self.config, ti=task_instance, dag=dag, dag_run=dagrun, datajob=datajob, emit_templates=False, - config=self.config, ) logger.debug(f"Emitted DataHub DataProcess Instance start: {dpi}") @@ -525,23 +531,119 @@ def on_dag_start(self, dag_run: "DagRun") -> None: return dataflow = AirflowGenerator.generate_dataflow( - cluster=self.config.cluster, + config=self.config, dag=dag, - capture_tags=self.config.capture_tags_info, - capture_owner=self.config.capture_ownership_info, ) dataflow.emit(self.emitter, callback=self._make_emit_callback()) + event: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityUrn=str(dataflow.urn), aspect=StatusClass(removed=False) + ) + self.emitter.emit(event) + + for task in dag.tasks: + task_urn = builder.make_data_job_urn_with_flow( + str(dataflow.urn), task.task_id + ) + event = MetadataChangeProposalWrapper( + entityUrn=task_urn, aspect=StatusClass(removed=False) + ) + self.emitter.emit(event) + # emit tags for tag in dataflow.tags: tag_urn = builder.make_tag_urn(tag) - event: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + event = MetadataChangeProposalWrapper( entityUrn=tag_urn, aspect=StatusClass(removed=False) ) - self.emitter.emit(event) + browse_path_v2_event: MetadataChangeProposalWrapper = ( + MetadataChangeProposalWrapper( + entityUrn=str(dataflow.urn), + aspect=BrowsePathsV2Class( + path=[BrowsePathEntryClass(str(dag.dag_id))], + ), + ) + ) + self.emitter.emit(browse_path_v2_event) + + if dag.dag_id == _DATAHUB_CLEANUP_DAG: + assert self.graph + + logger.debug("Initiating the cleanup of obsselete data from datahub") + + # get all ingested dataflow and datajob + ingested_dataflow_urns = list( + self.graph.get_urns_by_filter( + platform="airflow", + entity_types=["dataFlow"], + ) + ) + ingested_datajob_urns = list( + self.graph.get_urns_by_filter( + platform="airflow", entity_types=["dataJob"] + ) + ) + + # filter the ingested dataflow and datajob based on the cluster + filtered_ingested_dataflow_urns: List = [] + filtered_ingested_datajob_urns: List = [] + + for ingested_dataflow_urn in ingested_dataflow_urns: + data_flow_aspect = self.graph.get_aspect( + entity_urn=ingested_dataflow_urn, aspect_type=DataFlowKeyClass + ) + if ( + data_flow_aspect is not None + and data_flow_aspect.flowId != _DATAHUB_CLEANUP_DAG + and data_flow_aspect is not None + and data_flow_aspect.cluster == self.config.cluster + ): + filtered_ingested_dataflow_urns.append(ingested_dataflow_urn) + + for ingested_datajob_urn in ingested_datajob_urns: + data_job_aspect = self.graph.get_aspect( + entity_urn=ingested_datajob_urn, aspect_type=DataJobKeyClass + ) + if ( + data_job_aspect is not None + and data_job_aspect.flow in filtered_ingested_dataflow_urns + ): + filtered_ingested_datajob_urns.append(ingested_datajob_urn) + + # get all airflow dags + all_airflow_dags = SerializedDagModel.read_all_dags().values() + + airflow_flow_urns: List = [] + airflow_job_urns: List = [] + + for dag in all_airflow_dags: + flow_urn = builder.make_data_flow_urn( + orchestrator="airflow", + flow_id=dag.dag_id, + cluster=self.config.cluster, + ) + airflow_flow_urns.append(flow_urn) + + for task in dag.tasks: + airflow_job_urns.append( + builder.make_data_job_urn_with_flow(str(flow_urn), task.task_id) + ) + + obsolete_pipelines = set(filtered_ingested_dataflow_urns) - set( + airflow_flow_urns + ) + obsolete_tasks = set(filtered_ingested_datajob_urns) - set(airflow_job_urns) + + obsolete_urns = obsolete_pipelines.union(obsolete_tasks) + + asyncio.run(self._soft_delete_obsolete_urns(obsolete_urns=obsolete_urns)) + + logger.debug(f"total pipelines removed = {len(obsolete_pipelines)}") + logger.debug(f"total tasks removed = {len(obsolete_tasks)}") + if HAS_AIRFLOW_DAG_LISTENER_API: @hookimpl @@ -578,3 +680,13 @@ def on_dataset_changed(self, dataset: "Dataset") -> None: logger.debug( f"DataHub listener got notification about dataset change for {dataset}" ) + + async def _soft_delete_obsolete_urns(self, obsolete_urns): + delete_tasks = [self._delete_obsolete_data(urn) for urn in obsolete_urns] + await asyncio.gather(*delete_tasks) + + async def _delete_obsolete_data(self, obsolete_urn): + assert self.graph + + if self.graph.exists(str(obsolete_urn)): + self.graph.soft_delete_entity(str(obsolete_urn)) diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py index 628300d45d2fd..ace7669bfa998 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin_v22.py @@ -106,10 +106,8 @@ def datahub_task_status_callback(context, status): ) dataflow = AirflowGenerator.generate_dataflow( - cluster=config.cluster, + config=config, dag=dag, - capture_tags=config.capture_tags_info, - capture_owner=config.capture_ownership_info, ) task.log.info(f"Emitting Datahub Dataflow: {dataflow}") dataflow.emit(emitter, callback=_make_emit_callback(task.log)) @@ -139,13 +137,12 @@ def datahub_task_status_callback(context, status): if config.capture_executions: dpi = AirflowGenerator.run_datajob( emitter=emitter, - cluster=config.cluster, + config=config, ti=ti, dag=dag, dag_run=context["dag_run"], datajob=datajob, start_timestamp_millis=int(ti.start_date.timestamp() * 1000), - config=config, ) task.log.info(f"Emitted Start Datahub Dataprocess Instance: {dpi}") @@ -207,13 +204,12 @@ def datahub_pre_execution(context): if config.capture_executions: dpi = AirflowGenerator.run_datajob( emitter=emitter, - cluster=config.cluster, + config=config, ti=ti, dag=dag, dag_run=context["dag_run"], datajob=datajob, start_timestamp_millis=int(ti.start_date.timestamp() * 1000), - config=config, ) task.log.info(f"Emitting Datahub Dataprocess Instance: {dpi}") diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/lineage/_lineage_core.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/lineage/_lineage_core.py index 43e62c9f65f45..638458b0efd6a 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/lineage/_lineage_core.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/lineage/_lineage_core.py @@ -37,10 +37,8 @@ def send_lineage_to_datahub( emitter = hook.make_emitter() dataflow = AirflowGenerator.generate_dataflow( - cluster=config.cluster, + config=config, dag=dag, - capture_tags=config.capture_tags_info, - capture_owner=config.capture_ownership_info, ) dataflow.emit(emitter) operator.log.info(f"Emitted from Lineage: {dataflow}") @@ -68,7 +66,7 @@ def send_lineage_to_datahub( dpi = AirflowGenerator.run_datajob( emitter=emitter, - cluster=config.cluster, + config=config, ti=ti, dag=dag, dag_run=dag_run, diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/snowflake_operator.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/snowflake_operator.py index 347d0f88b0cd0..aac141ce31063 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/snowflake_operator.py +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/snowflake_operator.py @@ -6,12 +6,21 @@ SNOWFLAKE_COST_TABLE = "costs" SNOWFLAKE_PROCESSED_TABLE = "processed_costs" + +def _fake_snowflake_execute(*args, **kwargs): + raise ValueError("mocked snowflake execute to not run queries") + + with DAG( "snowflake_operator", start_date=datetime(2023, 1, 1), schedule_interval=None, catchup=False, ) as dag: + # HACK: We don't want to send real requests to Snowflake. As a workaround, + # we can simply monkey-patch the operator. + SnowflakeOperator.execute = _fake_snowflake_execute # type: ignore + transform_cost_table = SnowflakeOperator( snowflake_conn_id="my_snowflake", task_id="transform_cost_table", diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json index a21df71efcdac..ca0d9d04c82a9 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json @@ -38,6 +38,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -112,10 +113,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -123,10 +126,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableB,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableB", + "origin": "DEV" } } }, @@ -134,10 +139,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,cloud.mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "cloud.mydb.schema.tableC", + "origin": "PROD" } } }, @@ -145,10 +152,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -156,10 +165,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -167,10 +178,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableE,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableE", + "origin": "PROD" } } }, @@ -190,6 +203,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -264,10 +278,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -275,10 +291,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableB,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableB", + "origin": "DEV" } } }, @@ -286,10 +304,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,cloud.mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "cloud.mydb.schema.tableC", + "origin": "PROD" } } }, @@ -297,10 +317,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -308,10 +330,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -319,10 +343,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableE,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableE", + "origin": "PROD" } } }, @@ -342,6 +368,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -388,7 +415,7 @@ "name": "basic_iolets_run_data_task_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1701222667932, + "time": 1717180290951, "actor": "urn:li:corpuser:datahub" } } @@ -440,10 +467,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -451,10 +480,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableB,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableB", + "origin": "DEV" } } }, @@ -462,10 +493,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,cloud.mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "cloud.mydb.schema.tableC", + "origin": "PROD" } } }, @@ -473,10 +506,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -484,10 +519,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -495,10 +532,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableE,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableE", + "origin": "PROD" } } }, @@ -509,7 +548,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701222667932, + "timestampMillis": 1717180290951, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -526,7 +565,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701222668122, + "timestampMillis": 1717180291140, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json index 6116722350541..f13e9bd3dac07 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json @@ -39,6 +39,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -110,10 +111,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -121,10 +124,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -132,10 +137,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -155,6 +162,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -226,10 +234,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -237,10 +247,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -248,10 +260,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -271,6 +285,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -317,7 +332,7 @@ "name": "simple_dag_task_1_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1701222595752, + "time": 1717180227827, "actor": "urn:li:corpuser:datahub" } } @@ -366,10 +381,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -377,10 +394,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -388,10 +407,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -402,7 +423,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701222595752, + "timestampMillis": 1717180227827, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -419,7 +440,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701222595962, + "timestampMillis": 1717180228022, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -472,6 +493,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -550,6 +572,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -628,6 +651,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -674,7 +698,7 @@ "name": "simple_dag_run_another_data_task_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1701222599804, + "time": 1717180231676, "actor": "urn:li:corpuser:datahub" } } @@ -699,7 +723,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701222599804, + "timestampMillis": 1717180231676, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -716,7 +740,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701222599959, + "timestampMillis": 1717180231824, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json index 8b1bad5b55874..e2f738d19d89a 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json @@ -38,6 +38,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -56,6 +57,43 @@ } } }, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,basic_iolets,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,basic_iolets,prod)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "basic_iolets" + } + ] + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)", @@ -75,7 +113,7 @@ "downstream_task_ids": "[]", "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableB', env='DEV', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableC', env='PROD', platform_instance='cloud'), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]", "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableE', env='PROD', platform_instance=None)]", - "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"dag\": \"<>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"task_id\": \"run_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<>\", \"_lock_for_execution\": true, \"_log\": \"<>\", \"append_env\": false, \"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<>\", \"task_group\": \"<>\", \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"is_setup\": false, \"is_teardown\": false, \"mapped\": false, \"operator_class\": \"airflow.operators.bash.BashOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=basic_iolets&_flt_3_task_id=run_data_task", "name": "run_data_task", @@ -113,10 +151,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,cloud.mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "cloud.mydb.schema.tableC", + "origin": "PROD" } } }, @@ -124,10 +164,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -135,10 +177,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableB,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableB", + "origin": "DEV" } } }, @@ -146,10 +190,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -157,10 +203,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -168,10 +216,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableE,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableE", + "origin": "PROD" } } }, @@ -191,6 +241,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -228,16 +279,16 @@ "state": "running", "operator": "BashOperator", "priority_weight": "1", - "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1", + "log_url": "http://airflow.example.com/dags/basic_iolets/grid?dag_run_id=manual_run_test&task_id=run_data_task&map_index=-1&tab=logs", "orchestrator": "airflow", "dag_id": "basic_iolets", "task_id": "run_data_task" }, - "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1", + "externalUrl": "http://airflow.example.com/dags/basic_iolets/grid?dag_run_id=manual_run_test&task_id=run_data_task&map_index=-1&tab=logs", "name": "basic_iolets_run_data_task_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1701223416947, + "time": 1718733614956, "actor": "urn:li:corpuser:datahub" } } @@ -289,10 +340,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,cloud.mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "cloud.mydb.schema.tableC", + "origin": "PROD" } } }, @@ -300,10 +353,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -311,10 +366,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableB,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableB", + "origin": "DEV" } } }, @@ -322,10 +379,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -333,10 +392,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -344,10 +405,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableE,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableE", + "origin": "PROD" } } }, @@ -358,7 +421,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701223416947, + "timestampMillis": 1717179624988, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -375,14 +438,14 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1714671978982, + "timestampMillis": 1717179625524, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" }, "actor": "urn:li:corpuser:airflow", "operationType": "CREATE", - "lastUpdatedTimestamp": 1714671978982 + "lastUpdatedTimestamp": 1717179625524 } } }, @@ -393,14 +456,14 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1714671978991, + "timestampMillis": 1717179625547, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" }, "actor": "urn:li:corpuser:airflow", "operationType": "CREATE", - "lastUpdatedTimestamp": 1714671978991 + "lastUpdatedTimestamp": 1717179625547 } } }, @@ -423,7 +486,7 @@ "downstream_task_ids": "[]", "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableB', env='DEV', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableC', env='PROD', platform_instance='cloud'), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]", "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableE', env='PROD', platform_instance=None)]", - "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"dag\": \"<>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"task_id\": \"run_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<>\", \"_lock_for_execution\": true, \"_log\": \"<>\", \"append_env\": false, \"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<>\", \"task_group\": \"<>\", \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"is_setup\": false, \"is_teardown\": false, \"mapped\": false, \"operator_class\": \"airflow.operators.bash.BashOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=basic_iolets&_flt_3_task_id=run_data_task", "name": "run_data_task", @@ -461,10 +524,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,cloud.mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "cloud.mydb.schema.tableC", + "origin": "PROD" } } }, @@ -472,10 +537,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -483,10 +550,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableB,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableB", + "origin": "DEV" } } }, @@ -494,10 +563,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -505,10 +576,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -516,10 +589,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableE,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableE", + "origin": "PROD" } } }, @@ -565,7 +640,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1714671979032, + "timestampMillis": 1717179625632, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json index 589cd32ae3eb7..7b1591bdf7308 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets_no_dag_listener.json @@ -38,6 +38,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -56,6 +57,43 @@ } } }, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,basic_iolets,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,basic_iolets,prod)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "basic_iolets" + } + ] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)", @@ -75,7 +113,7 @@ "downstream_task_ids": "[]", "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableB', env='DEV', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableC', env='PROD', platform_instance='cloud'), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]", "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableE', env='PROD', platform_instance=None)]", - "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"dag\": \"<>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"task_id\": \"run_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<>\", \"_log\": \"<>\", \"append_env\": false, \"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<>\", \"retry_exponential_backoff\": false, \"skip_exit_code\": 99, \"start_date\": \"<>\", \"task_group\": \"<>\", \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"mapped\": false, \"operator_class\": \"airflow.operators.bash.BashOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=basic_iolets&_flt_3_task_id=run_data_task", "name": "run_data_task", @@ -113,10 +151,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,cloud.mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "cloud.mydb.schema.tableC", + "origin": "PROD" } } }, @@ -124,10 +164,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -135,10 +177,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableB,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableB", + "origin": "DEV" } } }, @@ -146,10 +190,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -157,10 +203,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -168,10 +216,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableE,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableE", + "origin": "PROD" } } }, @@ -191,6 +241,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -237,7 +288,7 @@ "name": "basic_iolets_run_data_task_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1701223185349, + "time": 1717180006234, "actor": "urn:li:corpuser:datahub" } } @@ -289,10 +340,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,cloud.mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "cloud.mydb.schema.tableC", + "origin": "PROD" } } }, @@ -300,10 +353,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -311,10 +366,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableB,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableB", + "origin": "DEV" } } }, @@ -322,10 +379,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -333,10 +392,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -344,10 +405,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableE,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableE", + "origin": "PROD" } } }, @@ -358,7 +421,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701223185349, + "timestampMillis": 1717180006234, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -375,14 +438,14 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1714676628119, + "timestampMillis": 1717180006652, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" }, "actor": "urn:li:corpuser:airflow", "operationType": "CREATE", - "lastUpdatedTimestamp": 1714676628119 + "lastUpdatedTimestamp": 1717180006652 } } }, @@ -393,14 +456,14 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1714676628127, + "timestampMillis": 1717180006674, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" }, "actor": "urn:li:corpuser:airflow", "operationType": "CREATE", - "lastUpdatedTimestamp": 1714676628127 + "lastUpdatedTimestamp": 1717180006674 } } }, @@ -423,7 +486,7 @@ "downstream_task_ids": "[]", "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableB', env='DEV', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableC', env='PROD', platform_instance='cloud'), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]", "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableE', env='PROD', platform_instance=None)]", - "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"dag\": \"<>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"task_id\": \"run_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<>\", \"_log\": \"<>\", \"append_env\": false, \"bash_command\": \"echo 'This is where you might run your data tooling.'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"env\": \"DEV\", \"name\": \"mydb.schema.tableB\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableC\", \"platform\": \"snowflake\", \"platform_instance\": \"cloud\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}, {\"env\": \"PROD\", \"name\": \"mydb.schema.tableE\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<>\", \"retry_exponential_backoff\": false, \"skip_exit_code\": 99, \"start_date\": \"<>\", \"task_group\": \"<>\", \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"mapped\": false, \"operator_class\": \"airflow.operators.bash.BashOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=basic_iolets&_flt_3_task_id=run_data_task", "name": "run_data_task", @@ -461,10 +524,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,cloud.mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "cloud.mydb.schema.tableC", + "origin": "PROD" } } }, @@ -472,10 +537,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -483,10 +550,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableB,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableB", + "origin": "DEV" } } }, @@ -494,10 +563,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -505,10 +576,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -516,10 +589,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableE,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableE", + "origin": "PROD" } } }, @@ -539,6 +614,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -564,7 +640,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701223186055, + "timestampMillis": 1717180006942, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json index 653d8f7e30530..0828c5e5aa1f7 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag.json @@ -39,6 +39,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -57,6 +58,43 @@ } } }, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,simple_dag,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),task_1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,simple_dag,prod)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "simple_dag" + } + ] + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),task_1)", @@ -76,7 +114,7 @@ "downstream_task_ids": "['run_another_data_task']", "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]", "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None)]", - "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 1'\", \"dag\": \"<>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"task_id\": \"task_1\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<>\", \"_lock_for_execution\": true, \"_log\": \"<>\", \"append_env\": false, \"bash_command\": \"echo 'task 1'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [\"run_another_data_task\"], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<>\", \"task_group\": \"<>\", \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"['run_another_data_task']\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"is_setup\": false, \"is_teardown\": false, \"mapped\": false, \"operator_class\": \"airflow.operators.bash.BashOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=task_1", "name": "task_1", @@ -86,6 +124,17 @@ } } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),run_another_data_task)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),task_1)", @@ -111,10 +160,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -122,10 +173,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -133,10 +186,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -156,6 +211,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -193,16 +249,16 @@ "state": "running", "operator": "BashOperator", "priority_weight": "2", - "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1", + "log_url": "http://airflow.example.com/dags/simple_dag/grid?dag_run_id=manual_run_test&task_id=task_1&map_index=-1&tab=logs", "orchestrator": "airflow", "dag_id": "simple_dag", "task_id": "task_1" }, - "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1", + "externalUrl": "http://airflow.example.com/dags/simple_dag/grid?dag_run_id=manual_run_test&task_id=task_1&map_index=-1&tab=logs", "name": "simple_dag_task_1_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1701223349283, + "time": 1718733547259, "actor": "urn:li:corpuser:datahub" } } @@ -251,10 +307,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -262,10 +320,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -273,10 +333,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -287,7 +349,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701223349283, + "timestampMillis": 1717179559032, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -304,14 +366,14 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1714671938600, + "timestampMillis": 1717179559525, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" }, "actor": "urn:li:corpuser:airflow", "operationType": "CREATE", - "lastUpdatedTimestamp": 1714671938600 + "lastUpdatedTimestamp": 1717179559525 } } }, @@ -334,7 +396,7 @@ "downstream_task_ids": "['run_another_data_task']", "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]", "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None)]", - "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 1'\", \"dag\": \"<>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"task_id\": \"task_1\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<>\", \"_lock_for_execution\": true, \"_log\": \"<>\", \"append_env\": false, \"bash_command\": \"echo 'task 1'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [\"run_another_data_task\"], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<>\", \"task_group\": \"<>\", \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"['run_another_data_task']\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"is_setup\": false, \"is_teardown\": false, \"mapped\": false, \"operator_class\": \"airflow.operators.bash.BashOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=task_1", "name": "task_1", @@ -369,10 +431,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -380,10 +444,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -391,10 +457,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -414,6 +482,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -439,7 +508,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701223349928, + "timestampMillis": 1717179559610, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -471,7 +540,7 @@ "downstream_task_ids": "[]", "inlets": "[]", "outlets": "[]", - "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 2'\", \"dag\": \"<>\", \"task_id\": \"run_another_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<>\", \"_lock_for_execution\": true, \"_log\": \"<>\", \"append_env\": false, \"bash_command\": \"echo 'task 2'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [], \"outlets\": [], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<>\", \"task_group\": \"<>\", \"task_id\": \"run_another_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [\"task_1\"], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"is_setup\": false, \"is_teardown\": false, \"mapped\": false, \"operator_class\": \"airflow.operators.bash.BashOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"run_another_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"['task_1']\", \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=run_another_data_task", "name": "run_another_data_task", @@ -513,6 +582,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -550,16 +620,16 @@ "state": "running", "operator": "BashOperator", "priority_weight": "1", - "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1", + "log_url": "http://airflow.example.com/dags/simple_dag/grid?dag_run_id=manual_run_test&task_id=run_another_data_task&map_index=-1&tab=logs", "orchestrator": "airflow", "dag_id": "simple_dag", "task_id": "run_another_data_task" }, - "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1", + "externalUrl": "http://airflow.example.com/dags/simple_dag/grid?dag_run_id=manual_run_test&task_id=run_another_data_task&map_index=-1&tab=logs", "name": "simple_dag_run_another_data_task_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1701223355004, + "time": 1718733551439, "actor": "urn:li:corpuser:datahub" } } @@ -584,7 +654,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701223355004, + "timestampMillis": 1717179564453, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -613,7 +683,7 @@ "downstream_task_ids": "[]", "inlets": "[]", "outlets": "[]", - "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 2'\", \"dag\": \"<>\", \"task_id\": \"run_another_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<>\", \"_lock_for_execution\": true, \"_log\": \"<>\", \"append_env\": false, \"bash_command\": \"echo 'task 2'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [], \"outlets\": [], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<>\", \"retry_exponential_backoff\": false, \"skip_on_exit_code\": [99], \"start_date\": \"<>\", \"task_group\": \"<>\", \"task_id\": \"run_another_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [\"task_1\"], \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"is_setup\": false, \"is_teardown\": false, \"mapped\": false, \"operator_class\": \"airflow.operators.bash.BashOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"run_another_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"['task_1']\", \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=run_another_data_task", "name": "run_another_data_task", @@ -655,6 +725,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -680,7 +751,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701223355580, + "timestampMillis": 1717179564937, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json index da08d2addf7c9..c7cd245cc0f02 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_simple_dag_no_dag_listener.json @@ -39,6 +39,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -57,6 +58,43 @@ } } }, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,simple_dag,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,simple_dag,prod)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "simple_dag" + } + ] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),task_1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),task_1)", @@ -76,7 +114,7 @@ "downstream_task_ids": "['run_another_data_task']", "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]", "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None)]", - "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 1'\", \"dag\": \"<>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"task_id\": \"task_1\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<>\", \"_log\": \"<>\", \"append_env\": false, \"bash_command\": \"echo 'task 1'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [\"run_another_data_task\"], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<>\", \"retry_exponential_backoff\": false, \"skip_exit_code\": 99, \"start_date\": \"<>\", \"task_group\": \"<>\", \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"['run_another_data_task']\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"mapped\": false, \"operator_class\": \"airflow.operators.bash.BashOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=task_1", "name": "task_1", @@ -86,6 +124,17 @@ } } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),run_another_data_task)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),task_1)", @@ -111,10 +160,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -122,10 +173,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -133,10 +186,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -156,6 +211,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -202,7 +258,7 @@ "name": "simple_dag_task_1_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1701223113232, + "time": 1717179933913, "actor": "urn:li:corpuser:datahub" } } @@ -251,10 +307,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -262,10 +320,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -273,10 +333,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -287,7 +349,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701223113232, + "timestampMillis": 1717179933913, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -304,14 +366,14 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1714676586630, + "timestampMillis": 1717179934145, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" }, "actor": "urn:li:corpuser:airflow", "operationType": "CREATE", - "lastUpdatedTimestamp": 1714676586630 + "lastUpdatedTimestamp": 1717179934145 } } }, @@ -334,7 +396,7 @@ "downstream_task_ids": "['run_another_data_task']", "inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]", "outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None)]", - "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 1'\", \"dag\": \"<>\", \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"task_id\": \"task_1\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<>\", \"_log\": \"<>\", \"append_env\": false, \"bash_command\": \"echo 'task 1'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [\"run_another_data_task\"], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableA\", \"platform\": \"snowflake\"}, {\"_urn\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)\"}, {\"_urn\": \"urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)\"}], \"outlets\": [{\"env\": \"PROD\", \"name\": \"mydb.schema.tableD\", \"platform\": \"snowflake\"}], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<>\", \"retry_exponential_backoff\": false, \"skip_exit_code\": 99, \"start_date\": \"<>\", \"task_group\": \"<>\", \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [], \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"['run_another_data_task']\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"mapped\": false, \"operator_class\": \"airflow.operators.bash.BashOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"task_1\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=task_1", "name": "task_1", @@ -369,10 +431,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableA,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableA", + "origin": "PROD" } } }, @@ -380,10 +444,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableC", + "origin": "PROD" } } }, @@ -391,10 +457,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableD,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "mydb.schema.tableD", + "origin": "PROD" } } }, @@ -414,6 +482,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -439,7 +508,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701223113778, + "timestampMillis": 1717179934378, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -492,6 +561,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -529,7 +599,7 @@ "downstream_task_ids": "[]", "inlets": "[]", "outlets": "[]", - "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 2'\", \"dag\": \"<>\", \"task_id\": \"run_another_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<>\", \"_log\": \"<>\", \"append_env\": false, \"bash_command\": \"echo 'task 2'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [], \"outlets\": [], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<>\", \"retry_exponential_backoff\": false, \"skip_exit_code\": 99, \"start_date\": \"<>\", \"task_group\": \"<>\", \"task_id\": \"run_another_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [\"task_1\"], \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"mapped\": false, \"operator_class\": \"airflow.operators.bash.BashOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"run_another_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"['task_1']\", \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=run_another_data_task", "name": "run_another_data_task", @@ -571,6 +641,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -617,7 +688,7 @@ "name": "simple_dag_run_another_data_task_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1701223119777, + "time": 1717179938499, "actor": "urn:li:corpuser:datahub" } } @@ -642,7 +713,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701223119777, + "timestampMillis": 1717179938499, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -671,7 +742,7 @@ "downstream_task_ids": "[]", "inlets": "[]", "outlets": "[]", - "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"_BaseOperator__from_mapped\": false, \"_BaseOperator__init_kwargs\": {\"bash_command\": \"echo 'task 2'\", \"dag\": \"<>\", \"task_id\": \"run_another_data_task\"}, \"_BaseOperator__instantiated\": true, \"_dag\": \"<>\", \"_log\": \"<>\", \"append_env\": false, \"bash_command\": \"echo 'task 2'\", \"depends_on_past\": false, \"do_xcom_push\": true, \"downstream_task_ids\": [], \"email_on_failure\": true, \"email_on_retry\": true, \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"inlets\": [], \"outlets\": [], \"output_encoding\": \"utf-8\", \"owner\": \"airflow\", \"params\": \"<>\", \"pool\": \"default_pool\", \"pool_slots\": 1, \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_delay\": \"<>\", \"retry_exponential_backoff\": false, \"skip_exit_code\": 99, \"start_date\": \"<>\", \"task_group\": \"<>\", \"task_id\": \"run_another_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": [\"task_1\"], \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"mapped\": false, \"operator_class\": \"airflow.operators.bash.BashOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"run_another_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"['task_1']\", \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=run_another_data_task", "name": "run_another_data_task", @@ -713,6 +784,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -738,7 +810,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701223120456, + "timestampMillis": 1717179939057, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json index 331ecd353ba26..b819379395a04 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json @@ -38,6 +38,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -56,6 +57,43 @@ } } }, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,snowflake_operator,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,snowflake_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,snowflake_operator,prod)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "snowflake_operator" + } + ] + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,snowflake_operator,prod),transform_cost_table)", @@ -76,7 +114,7 @@ "downstream_task_ids": "[]", "inlets": "[]", "outlets": "[]", - "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}" + "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=snowflake_operator&_flt_3_task_id=transform_cost_table", "name": "transform_cost_table", @@ -165,10 +203,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,datahub_test_database.datahub_test_schema.costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "datahub_test_database.datahub_test_schema.costs", + "origin": "PROD" } } }, @@ -176,10 +216,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,datahub_test_database.datahub_test_schema.processed_costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "datahub_test_database.datahub_test_schema.processed_costs", + "origin": "PROD" } } }, @@ -199,6 +241,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -236,16 +279,16 @@ "state": "running", "operator": "SnowflakeOperator", "priority_weight": "1", - "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=snowflake_operator&map_index=-1", + "log_url": "http://airflow.example.com/dags/snowflake_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&map_index=-1&tab=logs", "orchestrator": "airflow", "dag_id": "snowflake_operator", "task_id": "transform_cost_table" }, - "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=snowflake_operator&map_index=-1", + "externalUrl": "http://airflow.example.com/dags/snowflake_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&map_index=-1&tab=logs", "name": "snowflake_operator_transform_cost_table_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1701223475050, + "time": 1718733682840, "actor": "urn:li:corpuser:datahub" } } @@ -293,10 +336,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,datahub_test_database.datahub_test_schema.costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "datahub_test_database.datahub_test_schema.costs", + "origin": "PROD" } } }, @@ -304,10 +349,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,datahub_test_database.datahub_test_schema.processed_costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "datahub_test_database.datahub_test_schema.processed_costs", + "origin": "PROD" } } }, @@ -318,7 +365,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701223475050, + "timestampMillis": 1717179684292, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -335,14 +382,14 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1714672017187, + "timestampMillis": 1717179684935, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" }, "actor": "urn:li:corpuser:airflow", "operationType": "CREATE", - "lastUpdatedTimestamp": 1714672017187 + "lastUpdatedTimestamp": 1717179684935 } } }, @@ -366,7 +413,7 @@ "downstream_task_ids": "[]", "inlets": "[]", "outlets": "[]", - "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}" + "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=snowflake_operator&_flt_3_task_id=transform_cost_table", "name": "transform_cost_table", @@ -455,10 +502,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,datahub_test_database.datahub_test_schema.costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "datahub_test_database.datahub_test_schema.costs", + "origin": "PROD" } } }, @@ -466,10 +515,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,datahub_test_database.datahub_test_schema.processed_costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:snowflake", + "name": "datahub_test_database.datahub_test_schema.processed_costs", + "origin": "PROD" } } }, @@ -489,6 +540,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -514,7 +566,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1701223476665, + "timestampMillis": 1717179685374, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json index 693e9b6120a1c..e7902d165051b 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator.json @@ -57,6 +57,43 @@ } } }, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,sqlite_operator,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),create_cost_table)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,sqlite_operator,prod)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "sqlite_operator" + } + ] + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),create_cost_table)", @@ -87,6 +124,39 @@ } } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_processed_costs)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),populate_cost_table)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),create_cost_table)", @@ -141,8 +211,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_costs)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -151,6 +221,19 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.costs", + "origin": "PROD" + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),create_cost_table)", @@ -205,16 +288,16 @@ "state": "running", "operator": "SqliteOperator", "priority_weight": "5", - "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=create_cost_table&dag_id=sqlite_operator&map_index=-1", + "log_url": "http://airflow.example.com/dags/sqlite_operator/grid?dag_run_id=manual_run_test&task_id=create_cost_table&map_index=-1&tab=logs", "orchestrator": "airflow", "dag_id": "sqlite_operator", "task_id": "create_cost_table" }, - "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=create_cost_table&dag_id=sqlite_operator&map_index=-1", + "externalUrl": "http://airflow.example.com/dags/sqlite_operator/grid?dag_run_id=manual_run_test&task_id=create_cost_table&map_index=-1&tab=logs", "name": "sqlite_operator_create_cost_table_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1716506459310, + "time": 1718715882139, "actor": "urn:li:corpuser:datahub" } } @@ -249,10 +332,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.costs", + "origin": "PROD" } } }, @@ -263,7 +348,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1716506459310, + "timestampMillis": 1717179743558, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -280,14 +365,14 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1716506459665, + "timestampMillis": 1717179743932, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" }, "actor": "urn:li:corpuser:airflow", "operationType": "CREATE", - "lastUpdatedTimestamp": 1716506459665 + "lastUpdatedTimestamp": 1717179743932 } } }, @@ -414,10 +499,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.costs", + "origin": "PROD" } } }, @@ -463,7 +550,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1716506459692, + "timestampMillis": 1717179743960, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -530,10 +617,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.costs", + "origin": "PROD" } } }, @@ -600,7 +689,7 @@ "name": "sqlite_operator_populate_cost_table_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1716506463946, + "time": 1718715886259, "actor": "urn:li:corpuser:datahub" } } @@ -635,10 +724,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.costs", + "origin": "PROD" } } }, @@ -649,7 +740,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1716506463946, + "timestampMillis": 1717179748679, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -666,14 +757,14 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1716506464455, + "timestampMillis": 1717179749258, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" }, "actor": "urn:li:corpuser:airflow", "operationType": "CREATE", - "lastUpdatedTimestamp": 1716506464455 + "lastUpdatedTimestamp": 1717179749258 } } }, @@ -731,10 +822,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.costs", + "origin": "PROD" } } }, @@ -780,7 +873,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1716506464494, + "timestampMillis": 1717179749324, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -904,10 +997,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.costs", + "origin": "PROD" } } }, @@ -915,10 +1010,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.processed_costs", + "origin": "PROD" } } }, @@ -976,16 +1073,16 @@ "state": "running", "operator": "SqliteOperator", "priority_weight": "3", - "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1", + "log_url": "http://airflow.example.com/dags/sqlite_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&map_index=-1&tab=logs", "orchestrator": "airflow", "dag_id": "sqlite_operator", "task_id": "transform_cost_table" }, - "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1", + "externalUrl": "http://airflow.example.com/dags/sqlite_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&map_index=-1&tab=logs", "name": "sqlite_operator_transform_cost_table_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1716506468706, + "time": 1718715894274, "actor": "urn:li:corpuser:datahub" } } @@ -1033,10 +1130,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.costs", + "origin": "PROD" } } }, @@ -1044,10 +1143,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.processed_costs", + "origin": "PROD" } } }, @@ -1058,7 +1159,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1716506468706, + "timestampMillis": 1717179757397, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -1075,14 +1176,14 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1716506469563, + "timestampMillis": 1717179758424, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" }, "actor": "urn:li:corpuser:airflow", "operationType": "CREATE", - "lastUpdatedTimestamp": 1716506469563 + "lastUpdatedTimestamp": 1717179758424 } } }, @@ -1253,10 +1354,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.costs", + "origin": "PROD" } } }, @@ -1264,10 +1367,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetKey", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.processed_costs", + "origin": "PROD" } } }, @@ -1313,7 +1418,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1716506469626, + "timestampMillis": 1717179758496, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -1358,6 +1463,19 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.costs", + "origin": "PROD" + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_costs)", @@ -1376,17 +1494,6 @@ } } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_costs)", @@ -1422,6 +1529,58 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.processed_costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.processed_costs", + "origin": "PROD" + } + } +}, { "entityType": "dataProcessInstance", "entityUrn": "urn:li:dataProcessInstance:07285de22276959612189d51336cc21a", @@ -1441,16 +1600,16 @@ "state": "running", "operator": "SqliteOperator", "priority_weight": "1", - "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_costs&dag_id=sqlite_operator&map_index=-1", + "log_url": "http://airflow.example.com/dags/sqlite_operator/grid?dag_run_id=manual_run_test&task_id=cleanup_costs&map_index=-1&tab=logs", "orchestrator": "airflow", "dag_id": "sqlite_operator", "task_id": "cleanup_costs" }, - "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_costs&dag_id=sqlite_operator&map_index=-1", + "externalUrl": "http://airflow.example.com/dags/sqlite_operator/grid?dag_run_id=manual_run_test&task_id=cleanup_costs&map_index=-1&tab=logs", "name": "sqlite_operator_cleanup_costs_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1716506477141, + "time": 1718733767964, "actor": "urn:li:corpuser:datahub" } } @@ -1469,26 +1628,28 @@ } }, { - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:07285de22276959612189d51336cc21a", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", "changeType": "UPSERT", - "aspectName": "dataProcessInstanceInput", + "aspectName": "datasetKey", "aspect": { "json": { - "inputs": [ - "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" - ] + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.processed_costs", + "origin": "PROD" } } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:07285de22276959612189d51336cc21a", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "dataProcessInstanceInput", "aspect": { "json": { - "removed": false + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" + ] } } }, @@ -1499,7 +1660,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1716506477141, + "timestampMillis": 1718733767964, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -1510,46 +1671,34 @@ } }, { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_costs)", + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:07285de22276959612189d51336cc21a", "changeType": "UPSERT", - "aspectName": "dataJobInfo", + "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "customProperties": { - "depends_on_past": "False", - "email": "None", - "label": "'cleanup_costs'", - "execution_timeout": "None", - "sla": "None", - "sql": "'\\n DROP TABLE costs\\n '", - "task_id": "'cleanup_costs'", - "trigger_rule": "", - "wait_for_downstream": "False", - "downstream_task_ids": "[]", - "inlets": "[]", - "outlets": "[]", - "datahub_sql_parser_error": "UnsupportedStatementTypeError: Can only generate column-level lineage for select-like inner statements, not (outer statement type: )", - "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n DROP TABLE costs\\n \"}", - "openlineage_run_facet_extractionError": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/ExtractionErrorRunFacet\", \"errors\": [{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"errorMessage\": \"Can only generate column-level lineage for select-like inner statements, not (outer statement type: )\", \"task\": \"datahub_sql_parser\"}], \"failedTasks\": 1, \"totalTasks\": 1}" + "timestampMillis": 1718733768638, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" }, - "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=cleanup_costs", - "name": "cleanup_costs", - "type": { - "string": "COMMAND" + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "airflow" } } } }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_costs)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_processed_costs)", "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { "json": { "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" + "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)" ], "outputDatasets": [], "inputDatajobs": [ @@ -1559,72 +1708,6 @@ } } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_costs)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:airflow", - "type": "DEVELOPER", - "source": { - "type": "SERVICE" - } - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:airflow" - } - } - } -}, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_costs)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [] - } - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:07285de22276959612189d51336cc21a", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRunEvent", - "aspect": { - "json": { - "timestampMillis": 1716506478016, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "status": "COMPLETE", - "result": { - "type": "SUCCESS", - "nativeResultType": "airflow" - } - } - } -}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_processed_costs)", @@ -1657,35 +1740,6 @@ } } }, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_processed_costs)", - "changeType": "UPSERT", - "aspectName": "dataJobInputOutput", - "aspect": { - "json": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)" - ], - "outputDatasets": [], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)" - ], - "fineGrainedLineages": [] - } - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_processed_costs)", @@ -1740,33 +1794,21 @@ "state": "running", "operator": "SqliteOperator", "priority_weight": "1", - "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_processed_costs&dag_id=sqlite_operator&map_index=-1", + "log_url": "http://airflow.example.com/dags/sqlite_operator/grid?dag_run_id=manual_run_test&task_id=cleanup_processed_costs&map_index=-1&tab=logs", "orchestrator": "airflow", "dag_id": "sqlite_operator", "task_id": "cleanup_processed_costs" }, - "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=cleanup_processed_costs&dag_id=sqlite_operator&map_index=-1", + "externalUrl": "http://airflow.example.com/dags/sqlite_operator/grid?dag_run_id=manual_run_test&task_id=cleanup_processed_costs&map_index=-1&tab=logs", "name": "sqlite_operator_cleanup_processed_costs_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1716506482495, + "time": 1718733773354, "actor": "urn:li:corpuser:datahub" } } } }, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:bab908abccf3cd6607b50fdaf3003372", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRelationships", - "aspect": { - "json": { - "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_processed_costs)", - "upstreamInstances": [] - } - } -}, { "entityType": "dataProcessInstance", "entityUrn": "urn:li:dataProcessInstance:bab908abccf3cd6607b50fdaf3003372", @@ -1781,13 +1823,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:bab908abccf3cd6607b50fdaf3003372", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "dataProcessInstanceRelationships", "aspect": { "json": { - "removed": false + "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_processed_costs)", + "upstreamInstances": [] } } }, @@ -1798,7 +1841,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1716506482495, + "timestampMillis": 1718733773354, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -1808,102 +1851,6 @@ } } }, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_processed_costs)", - "changeType": "UPSERT", - "aspectName": "dataJobInfo", - "aspect": { - "json": { - "customProperties": { - "depends_on_past": "False", - "email": "None", - "label": "'cleanup_processed_costs'", - "execution_timeout": "None", - "sla": "None", - "sql": "'\\n DROP TABLE processed_costs\\n '", - "task_id": "'cleanup_processed_costs'", - "trigger_rule": "", - "wait_for_downstream": "False", - "downstream_task_ids": "[]", - "inlets": "[]", - "outlets": "[]", - "datahub_sql_parser_error": "UnsupportedStatementTypeError: Can only generate column-level lineage for select-like inner statements, not (outer statement type: )", - "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n DROP TABLE processed_costs\\n \"}", - "openlineage_run_facet_extractionError": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/ExtractionErrorRunFacet\", \"errors\": [{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.12.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"errorMessage\": \"Can only generate column-level lineage for select-like inner statements, not (outer statement type: )\", \"task\": \"datahub_sql_parser\"}], \"failedTasks\": 1, \"totalTasks\": 1}" - }, - "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=cleanup_processed_costs", - "name": "cleanup_processed_costs", - "type": { - "string": "COMMAND" - } - } - } -}, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_processed_costs)", - "changeType": "UPSERT", - "aspectName": "dataJobInputOutput", - "aspect": { - "json": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)" - ], - "outputDatasets": [], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)" - ], - "fineGrainedLineages": [] - } - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_processed_costs)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:airflow", - "type": "DEVELOPER", - "source": { - "type": "SERVICE" - } - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:airflow" - } - } - } -}, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_processed_costs)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [] - } - } -}, { "entityType": "dataProcessInstance", "entityUrn": "urn:li:dataProcessInstance:bab908abccf3cd6607b50fdaf3003372", @@ -1911,7 +1858,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1716506483469, + "timestampMillis": 1718733774147, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json index 47f7cdca68d49..a9af068e2e4e9 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_sqlite_operator_no_dag_listener.json @@ -38,6 +38,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -56,6 +57,65 @@ } } }, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,sqlite_operator,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),create_cost_table)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,sqlite_operator,prod)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "sqlite_operator" + } + ] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),populate_cost_table)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),create_cost_table)", @@ -75,8 +135,7 @@ "wait_for_downstream": "False", "downstream_task_ids": "['populate_cost_table']", "inlets": "[]", - "outlets": "[]", - "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE TABLE IF NOT EXISTS costs (\\n id INTEGER PRIMARY KEY,\\n month TEXT NOT NULL,\\n total_cost REAL NOT NULL,\\n area REAL NOT NULL\\n )\\n \"}" + "outlets": "[]" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=create_cost_table", "name": "create_cost_table", @@ -139,17 +198,6 @@ } } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),create_cost_table)", @@ -166,6 +214,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -173,6 +222,41 @@ } } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_costs)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_processed_costs)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),create_cost_table)", @@ -212,7 +296,7 @@ "name": "sqlite_operator_create_cost_table_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1707253281415, + "time": 1717180072004, "actor": "urn:li:corpuser:datahub" } } @@ -243,17 +327,6 @@ } } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, { "entityType": "dataProcessInstance", "entityUrn": "urn:li:dataProcessInstance:fbeed1180fa0434e02ac6f75ace87869", @@ -261,7 +334,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1707253281415, + "timestampMillis": 1717180072004, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -271,24 +344,6 @@ } } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1714676666839, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:airflow", - "operationType": "CREATE", - "lastUpdatedTimestamp": 1714676666839 - } - } -}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),create_cost_table)", @@ -308,8 +363,7 @@ "wait_for_downstream": "False", "downstream_task_ids": "['populate_cost_table']", "inlets": "[]", - "outlets": "[]", - "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE TABLE IF NOT EXISTS costs (\\n id INTEGER PRIMARY KEY,\\n month TEXT NOT NULL,\\n total_cost REAL NOT NULL,\\n area REAL NOT NULL\\n )\\n \"}" + "outlets": "[]" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=create_cost_table", "name": "create_cost_table", @@ -319,103 +373,21 @@ } } }, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),create_cost_table)", - "changeType": "UPSERT", - "aspectName": "dataJobInputOutput", - "aspect": { - "json": { - "inputDatasets": [], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" - ], - "inputDatajobs": [], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)" - ], - "confidenceScore": 1.0 - } - ] - } - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "operation", "aspect": { "json": { - "removed": false + "timestampMillis": 1719864194882, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1719864194882 } } }, @@ -435,6 +407,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -460,7 +433,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1707253282244, + "timestampMillis": 1717180072275, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -512,6 +485,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -549,8 +523,7 @@ "wait_for_downstream": "False", "downstream_task_ids": "['transform_cost_table']", "inlets": "[]", - "outlets": "[]", - "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n INSERT INTO costs (id, month, total_cost, area)\\n VALUES\\n (1, '2021-01', 100, 10),\\n (2, '2021-02', 200, 20),\\n (3, '2021-03', 300, 30)\\n \"}" + "outlets": "[]" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=populate_cost_table", "name": "populate_cost_table", @@ -578,17 +551,6 @@ } } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),populate_cost_table)", @@ -605,6 +567,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -651,7 +614,7 @@ "name": "sqlite_operator_populate_cost_table_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1707253286225, + "time": 1717180078196, "actor": "urn:li:corpuser:datahub" } } @@ -673,66 +636,24 @@ "entityType": "dataProcessInstance", "entityUrn": "urn:li:dataProcessInstance:04e1badac1eacd1c41123d07f579fa92", "changeType": "UPSERT", - "aspectName": "dataProcessInstanceOutput", + "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "outputs": [ - "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" - ] + "timestampMillis": 1717180078196, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "STARTED", + "attempt": 1 } } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),populate_cost_table)", "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:04e1badac1eacd1c41123d07f579fa92", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRunEvent", - "aspect": { - "json": { - "timestampMillis": 1707253286225, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "status": "STARTED", - "attempt": 1 - } - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1714676669640, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:airflow", - "operationType": "CREATE", - "lastUpdatedTimestamp": 1714676669640 - } - } -}, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),populate_cost_table)", - "changeType": "UPSERT", - "aspectName": "dataJobInfo", + "aspectName": "dataJobInfo", "aspect": { "json": { "customProperties": { @@ -747,8 +668,7 @@ "wait_for_downstream": "False", "downstream_task_ids": "['transform_cost_table']", "inlets": "[]", - "outlets": "[]", - "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n INSERT INTO costs (id, month, total_cost, area)\\n VALUES\\n (1, '2021-01', 100, 10),\\n (2, '2021-02', 200, 20),\\n (3, '2021-03', 300, 30)\\n \"}" + "outlets": "[]" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=populate_cost_table", "name": "populate_cost_table", @@ -758,35 +678,6 @@ } } }, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),populate_cost_table)", - "changeType": "UPSERT", - "aspectName": "dataJobInputOutput", - "aspect": { - "json": { - "inputDatasets": [], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" - ], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),create_cost_table)" - ], - "fineGrainedLineages": [] - } - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),populate_cost_table)", @@ -803,6 +694,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -828,7 +720,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1707253287414, + "timestampMillis": 1717180078619, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -880,6 +772,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -917,8 +810,7 @@ "wait_for_downstream": "False", "downstream_task_ids": "['cleanup_costs', 'cleanup_processed_costs']", "inlets": "[]", - "outlets": "[]", - "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE TABLE IF NOT EXISTS processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}" + "outlets": "[]" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=transform_cost_table", "name": "transform_cost_table", @@ -1005,28 +897,6 @@ } } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)", @@ -1043,6 +913,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -1063,315 +934,106 @@ }, { "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:64e5ff8f552e857b607832731e09808b", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceProperties", - "aspect": { - "json": { - "customProperties": { - "run_id": "manual_run_test", - "duration": "", - "start_date": "", - "end_date": "", - "execution_date": "2023-09-27 21:34:38+00:00", - "try_number": "0", - "max_tries": "0", - "external_executor_id": "None", - "state": "running", - "operator": "SqliteOperator", - "priority_weight": "3", - "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1", - "orchestrator": "airflow", - "dag_id": "sqlite_operator", - "task_id": "transform_cost_table" - }, - "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1", - "name": "sqlite_operator_transform_cost_table_manual_run_test", - "type": "BATCH_AD_HOC", - "created": { - "time": 1707253293513, - "actor": "urn:li:corpuser:datahub" - } - } - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:64e5ff8f552e857b607832731e09808b", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRelationships", - "aspect": { - "json": { - "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)", - "upstreamInstances": [] - } - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:64e5ff8f552e857b607832731e09808b", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceInput", - "aspect": { - "json": { - "inputs": [ - "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" - ] - } - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:64e5ff8f552e857b607832731e09808b", + "entityUrn": "urn:li:dataProcessInstance:04e1badac1eacd1c41123d07f579fa92", "changeType": "UPSERT", "aspectName": "dataProcessInstanceOutput", "aspect": { "json": { "outputs": [ - "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)" + "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" ] } } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, { "entityType": "dataProcessInstance", "entityUrn": "urn:li:dataProcessInstance:64e5ff8f552e857b607832731e09808b", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRunEvent", - "aspect": { - "json": { - "timestampMillis": 1707253293513, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "status": "STARTED", - "attempt": 1 - } - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1714676672665, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "actor": "urn:li:corpuser:airflow", - "operationType": "CREATE", - "lastUpdatedTimestamp": 1714676672665 - } - } -}, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)", - "changeType": "UPSERT", - "aspectName": "dataJobInfo", - "aspect": { - "json": { - "customProperties": { - "depends_on_past": "False", - "email": "None", - "label": "'transform_cost_table'", - "execution_timeout": "None", - "sla": "None", - "sql": "'\\n CREATE TABLE IF NOT EXISTS processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n '", - "task_id": "'transform_cost_table'", - "trigger_rule": "", - "wait_for_downstream": "False", - "downstream_task_ids": "['cleanup_costs', 'cleanup_processed_costs']", - "inlets": "[]", - "outlets": "[]", - "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE TABLE IF NOT EXISTS processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}" - }, - "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=transform_cost_table", - "name": "transform_cost_table", - "type": { - "string": "COMMAND" - } - } - } -}, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)", - "changeType": "UPSERT", - "aspectName": "dataJobInputOutput", - "aspect": { - "json": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)" - ], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),populate_cost_table)" - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD),id)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD),month)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD),total_cost)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD),area)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)", - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD),cost_per_area)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),id)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD),id)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),month)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD),month)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD),total_cost)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD),area)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),area)", - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD),total_cost)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD),cost_per_area)" - ], - "confidenceScore": 1.0 - } - ] + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "run_id": "manual_run_test", + "duration": "", + "start_date": "", + "end_date": "", + "execution_date": "2023-09-27 21:34:38+00:00", + "try_number": "0", + "max_tries": "0", + "external_executor_id": "None", + "state": "running", + "operator": "SqliteOperator", + "priority_weight": "3", + "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1", + "orchestrator": "airflow", + "dag_id": "sqlite_operator", + "task_id": "transform_cost_table" + }, + "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=sqlite_operator&map_index=-1", + "name": "sqlite_operator_transform_cost_table_manual_run_test", + "type": "BATCH_AD_HOC", + "created": { + "time": 1717180084642, + "actor": "urn:li:corpuser:datahub" + } } } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:64e5ff8f552e857b607832731e09808b", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "dataProcessInstanceRelationships", "aspect": { "json": { - "removed": false + "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)", + "upstreamInstances": [] } } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:64e5ff8f552e857b607832731e09808b", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "removed": false + "timestampMillis": 1717180084642, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "STARTED", + "attempt": 1 + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "depends_on_past": "False", + "email": "None", + "label": "'transform_cost_table'", + "execution_timeout": "None", + "sla": "None", + "sql": "'\\n CREATE TABLE IF NOT EXISTS processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n '", + "task_id": "'transform_cost_table'", + "trigger_rule": "", + "wait_for_downstream": "False", + "downstream_task_ids": "['cleanup_costs', 'cleanup_processed_costs']", + "inlets": "[]", + "outlets": "[]" + }, + "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=transform_cost_table", + "name": "transform_cost_table", + "type": { + "string": "COMMAND" + } } } }, @@ -1391,6 +1053,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -1416,7 +1079,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1707253295443, + "timestampMillis": 1717180085266, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -1468,6 +1131,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -1505,10 +1169,7 @@ "wait_for_downstream": "False", "downstream_task_ids": "[]", "inlets": "[]", - "outlets": "[]", - "datahub_sql_parser_error": "Can only generate column-level lineage for select-like inner statements, not (outer statement type: )", - "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n DROP TABLE costs\\n \"}", - "openlineage_run_facet_extractionError": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/ExtractionErrorRunFacet\", \"errors\": [{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"errorMessage\": \"Can only generate column-level lineage for select-like inner statements, not (outer statement type: )\", \"task\": \"datahub_sql_parser\"}], \"failedTasks\": 1, \"totalTasks\": 1}" + "outlets": "[]" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=cleanup_costs", "name": "cleanup_costs", @@ -1536,17 +1197,6 @@ } } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_costs)", @@ -1563,6 +1213,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -1609,7 +1260,7 @@ "name": "sqlite_operator_cleanup_costs_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1707253301697, + "time": 1717180091148, "actor": "urn:li:corpuser:datahub" } } @@ -1627,30 +1278,6 @@ } } }, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:07285de22276959612189d51336cc21a", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceInput", - "aspect": { - "json": { - "inputs": [ - "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" - ] - } - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, { "entityType": "dataProcessInstance", "entityUrn": "urn:li:dataProcessInstance:07285de22276959612189d51336cc21a", @@ -1658,7 +1285,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1707253301697, + "timestampMillis": 1717180091148, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -1687,10 +1314,7 @@ "wait_for_downstream": "False", "downstream_task_ids": "[]", "inlets": "[]", - "outlets": "[]", - "datahub_sql_parser_error": "Can only generate column-level lineage for select-like inner statements, not (outer statement type: )", - "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n DROP TABLE costs\\n \"}", - "openlineage_run_facet_extractionError": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/ExtractionErrorRunFacet\", \"errors\": [{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"errorMessage\": \"Can only generate column-level lineage for select-like inner statements, not (outer statement type: )\", \"task\": \"datahub_sql_parser\"}], \"failedTasks\": 1, \"totalTasks\": 1}" + "outlets": "[]" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=cleanup_costs", "name": "cleanup_costs", @@ -1700,35 +1324,6 @@ } } }, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_costs)", - "changeType": "UPSERT", - "aspectName": "dataJobInputOutput", - "aspect": { - "json": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" - ], - "outputDatasets": [], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)" - ], - "fineGrainedLineages": [] - } - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_costs)", @@ -1745,6 +1340,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -1770,7 +1366,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1707253303779, + "timestampMillis": 1717180091923, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -1783,6 +1379,19 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sqlite", + "name": "public.processed_costs", + "origin": "PROD" + } + } +}, { "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(airflow,sqlite_operator,prod)", @@ -1822,6 +1431,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -1859,10 +1469,7 @@ "wait_for_downstream": "False", "downstream_task_ids": "[]", "inlets": "[]", - "outlets": "[]", - "datahub_sql_parser_error": "Can only generate column-level lineage for select-like inner statements, not (outer statement type: )", - "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n DROP TABLE processed_costs\\n \"}", - "openlineage_run_facet_extractionError": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/ExtractionErrorRunFacet\", \"errors\": [{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"errorMessage\": \"Can only generate column-level lineage for select-like inner statements, not (outer statement type: )\", \"task\": \"datahub_sql_parser\"}], \"failedTasks\": 1, \"totalTasks\": 1}" + "outlets": "[]" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=cleanup_processed_costs", "name": "cleanup_processed_costs", @@ -1872,6 +1479,19 @@ } } }, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:64e5ff8f552e857b607832731e09808b", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "json": { + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" + ] + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_processed_costs)", @@ -1891,13 +1511,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:64e5ff8f552e857b607832731e09808b", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "dataProcessInstanceOutput", "aspect": { "json": { - "removed": false + "outputs": [ + "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)" + ] } } }, @@ -1917,6 +1539,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -1963,7 +1586,7 @@ "name": "sqlite_operator_cleanup_processed_costs_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1707253308368, + "time": 1717180096108, "actor": "urn:li:corpuser:datahub" } } @@ -1985,12 +1608,16 @@ "entityType": "dataProcessInstance", "entityUrn": "urn:li:dataProcessInstance:bab908abccf3cd6607b50fdaf3003372", "changeType": "UPSERT", - "aspectName": "dataProcessInstanceInput", + "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "inputs": [ - "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)" - ] + "timestampMillis": 1717180096108, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "STARTED", + "attempt": 1 } } }, @@ -1998,27 +1625,17 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:bab908abccf3cd6607b50fdaf3003372", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRunEvent", + "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1707253308368, + "timestampMillis": 1719864203487, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" }, - "status": "STARTED", - "attempt": 1 + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1719864203487 } } }, @@ -2041,10 +1658,7 @@ "wait_for_downstream": "False", "downstream_task_ids": "[]", "inlets": "[]", - "outlets": "[]", - "datahub_sql_parser_error": "Can only generate column-level lineage for select-like inner statements, not (outer statement type: )", - "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n DROP TABLE processed_costs\\n \"}", - "openlineage_run_facet_extractionError": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/ExtractionErrorRunFacet\", \"errors\": [{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.2.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/BaseFacet\", \"errorMessage\": \"Can only generate column-level lineage for select-like inner statements, not (outer statement type: )\", \"task\": \"datahub_sql_parser\"}], \"failedTasks\": 1, \"totalTasks\": 1}" + "outlets": "[]" }, "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=sqlite_operator&_flt_3_task_id=cleanup_processed_costs", "name": "cleanup_processed_costs", @@ -2054,35 +1668,6 @@ } } }, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_processed_costs)", - "changeType": "UPSERT", - "aspectName": "dataJobInputOutput", - "aspect": { - "json": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)" - ], - "outputDatasets": [], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),transform_cost_table)" - ], - "fineGrainedLineages": [] - } - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,sqlite_operator,prod),cleanup_processed_costs)", @@ -2099,6 +1684,7 @@ } } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:airflow" @@ -2124,7 +1710,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1707253310722, + "timestampMillis": 1717180096993, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -2136,5 +1722,31 @@ } } } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:bab908abccf3cd6607b50fdaf3003372", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "json": { + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.processed_costs,PROD)" + ] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:07285de22276959612189d51336cc21a", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "json": { + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:sqlite,public.costs,PROD)" + ] + } + } } ] \ No newline at end of file diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py index 005969aeba732..9ea822edeef81 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py @@ -255,7 +255,7 @@ def check_golden_file( update_golden=update_golden, copy_output=False, ignore_paths=ignore_paths, - ignore_order=False, + ignore_order=True, ) @@ -375,6 +375,8 @@ def test_airflow_plugin( # TODO: If we switched to Git urls, maybe we could get this to work consistently. r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['datahub_sql_parser_error'\]", r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['openlineage_.*'\]", + r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['log_url'\]", + r"root\[\d+\]\['aspect'\]\['json'\]\['externalUrl'\]", ], ) diff --git a/metadata-ingestion-modules/airflow-plugin/tests/unit/test_airflow.py b/metadata-ingestion-modules/airflow-plugin/tests/unit/test_airflow.py index c88f4d77b7aeb..36696d48cdaf7 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/unit/test_airflow.py +++ b/metadata-ingestion-modules/airflow-plugin/tests/unit/test_airflow.py @@ -365,13 +365,13 @@ def test_lineage_backend(mock_emit, inlets, outlets, capture_executions): == "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableProduced,PROD)" ) - assert mock_emitter.method_calls[5].args[0].aspectName == "status" + assert mock_emitter.method_calls[5].args[0].aspectName == "datasetKey" assert ( mock_emitter.method_calls[5].args[0].entityUrn == "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableConsumed,PROD)" ) - assert mock_emitter.method_calls[6].args[0].aspectName == "status" + assert mock_emitter.method_calls[6].args[0].aspectName == "datasetKey" assert ( mock_emitter.method_calls[6].args[0].entityUrn == "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableProduced,PROD)" @@ -423,12 +423,12 @@ def test_lineage_backend(mock_emit, inlets, outlets, capture_executions): mock_emitter.method_calls[12].args[0].entityUrn == "urn:li:dataProcessInstance:5e274228107f44cc2dd7c9782168cc29" ) - assert mock_emitter.method_calls[13].args[0].aspectName == "status" + assert mock_emitter.method_calls[13].args[0].aspectName == "datasetKey" assert ( mock_emitter.method_calls[13].args[0].entityUrn == "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableConsumed,PROD)" ) - assert mock_emitter.method_calls[14].args[0].aspectName == "status" + assert mock_emitter.method_calls[14].args[0].aspectName == "datasetKey" assert ( mock_emitter.method_calls[14].args[0].entityUrn == "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableProduced,PROD)" diff --git a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/__init__.py b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/__init__.py index 1ecfc362ceb4e..1c7d60666a085 100644 --- a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/__init__.py +++ b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/__init__.py @@ -1,6 +1,6 @@ # Published at https://pypi.org/project/acryl-datahub/. __package_name__ = "acryl-datahub-dagster-plugin" -__version__ = "0.0.0.dev0" +__version__ = "1!0.0.0.dev0" def is_dev_mode() -> bool: diff --git a/metadata-ingestion/cli-ingestion.md b/metadata-ingestion/cli-ingestion.md index b15dd2a501995..6a6dd65b3c98e 100644 --- a/metadata-ingestion/cli-ingestion.md +++ b/metadata-ingestion/cli-ingestion.md @@ -1,56 +1,108 @@ # CLI Ingestion -## Installing the CLI +Batch ingestion involves extracting metadata from a source system in bulk. Typically, this happens on a predefined schedule using the [Metadata Ingestion](../docs/components.md#ingestion-framework) framework. +The metadata that is extracted includes point-in-time instances of dataset, chart, dashboard, pipeline, user, group, usage, and task metadata. -Make sure you have installed DataHub CLI before following this guide. +## Installing DataHub CLI -```shell -# Requires Python 3.8+ +:::note Required Python Version +Installing DataHub CLI requires Python 3.6+. +::: + +Run the following commands in your terminal: + +``` python3 -m pip install --upgrade pip wheel setuptools python3 -m pip install --upgrade acryl-datahub -# validate that the install was successful -datahub version -# If you see "command not found", try running this instead: python3 -m datahub version +python3 -m datahub version ``` +Your command line should return the proper version of DataHub upon executing these commands successfully. + + Check out the [CLI Installation Guide](../docs/cli.md#installation) for more installation options and troubleshooting tips. -After that, install the required plugin for the ingestion. + +## Installing Connector Plugins + +Our CLI follows a plugin architecture. You must install connectors for different data sources individually. +For a list of all supported data sources, see [the open source docs](../docs/cli.md#sources). +Once you've found the connectors you care about, simply install them using `pip install`. +For example, to install the `mysql` connector, you can run ```shell -pip install 'acryl-datahub[datahub-rest]' # install the required plugin +pip install --upgrade 'acryl-datahub[mysql]' ``` - Check out the [alternative installation options](../docs/cli.md#alternate-installation-options) for more reference. ## Configuring a Recipe -Create a `recipe.yml` file that defines the source and sink for metadata, as shown below. +Create a [Recipe](recipe_overview.md) yaml file that defines the source and sink for metadata, as shown below. ```yaml -# recipe.yml +# example-recipe.yml + +# MySQL source configuration source: - type: + type: mysql config: - option_1: - ... + username: root + password: password + host_port: localhost:3306 +# Recipe sink configuration. sink: - type: + type: "datahub-rest" config: - ... + server: "https://.acryl.io/gms" + token: ``` +The **source** configuration block defines where to extract metadata from. This can be an OLTP database system, a data warehouse, or something as simple as a file. Each source has custom configuration depending on what is required to access metadata from the source. To see configurations required for each supported source, refer to the [Sources](source_overview.md) documentation. + +The **sink** configuration block defines where to push metadata into. Each sink type requires specific configurations, the details of which are detailed in the [Sinks](sink_overview.md) documentation. + +To configure your instance of DataHub as the destination for ingestion, set the "server" field of your recipe to point to your Acryl instance's domain suffixed by the path `/gms`, as shown below. +A complete example of a DataHub recipe file, which reads from MySQL and writes into a DataHub instance: For more information and examples on configuring recipes, please refer to [Recipes](recipe_overview.md). -## Ingesting Metadata -You can run ingestion using `datahub ingest` like below. +### Using Recipes with Authentication +In Acryl DataHub deployments, only the `datahub-rest` sink is supported, which simply means that metadata will be pushed to the REST endpoints exposed by your DataHub instance. The required configurations for this sink are + +1. **server**: the location of the REST API exposed by your instance of DataHub +2. **token**: a unique API key used to authenticate requests to your instance's REST API + +The token can be retrieved by logging in as admin. You can go to Settings page and generate a Personal Access Token with your desired expiration date. + +

+ +

+ +

+ +

+ +:::info Secure Your API Key +Please keep Your API key secure & avoid sharing it. +If you are on Acryl Cloud and your key is compromised for any reason, please reach out to the Acryl team at support@acryl.io. +::: + +## Ingesting Metadata +The final step requires invoking the DataHub CLI to ingest metadata based on your recipe configuration file. +To do so, simply run `datahub ingest` with a pointer to your YAML recipe file: ```shell datahub ingest -c ``` +## Scheduling Ingestion + +Ingestion can either be run in an ad-hoc manner by a system administrator or scheduled for repeated executions. Most commonly, ingestion will be run on a daily cadence. +To schedule your ingestion job, we recommend using a job schedule like [Apache Airflow](https://airflow.apache.org/). In cases of simpler deployments, a CRON job scheduled on an always-up machine can also work. +Note that each source system will require a separate recipe file. This allows you to schedule ingestion from different sources independently or together. +Learn more about scheduling ingestion in the [Scheduling Ingestion Guide](/metadata-ingestion/schedule_docs/intro.md). + ## Reference Please refer the following pages for advanced guids on CLI ingestion. @@ -58,9 +110,11 @@ Please refer the following pages for advanced guids on CLI ingestion. - [Reference for `datahub ingest` command](../docs/cli.md#ingest) - [UI Ingestion Guide](../docs/ui-ingestion.md) -:::Tip Compatibility +:::tip Compatibility + DataHub server uses a 3 digit versioning scheme, while the CLI uses a 4 digit scheme. For example, if you're using DataHub server version 0.10.0, you should use CLI version 0.10.0.x, where x is a patch version. We do this because we do CLI releases at a much higher frequency than server releases, usually every few days vs twice a month. For ingestion sources, any breaking changes will be highlighted in the [release notes](../docs/how/updating-datahub.md). When fields are deprecated or otherwise changed, we will try to maintain backwards compatibility for two server releases, which is about 4-6 weeks. The CLI will also print warnings whenever deprecated options are used. ::: + diff --git a/metadata-ingestion/developing.md b/metadata-ingestion/developing.md index c0d004e961059..e0dbc7c8d4b14 100644 --- a/metadata-ingestion/developing.md +++ b/metadata-ingestion/developing.md @@ -34,7 +34,30 @@ cd metadata-ingestion-modules/airflow-plugin ../../gradlew :metadata-ingestion-modules:airflow-plugin:installDev source venv/bin/activate datahub version # should print "DataHub CLI version: unavailable (installed in develop mode)" + +# start the airflow web server +export AIRFLOW_HOME=~/airflow +airflow webserver --port 8090 -d + +# start the airflow scheduler +airflow scheduler + +# access the airflow service and run any of the DAG +# open http://localhost:8090/ +# select any DAG and click on the `play arrow` button to start the DAG + +# add the debug lines in the codebase, i.e. in ./src/datahub_airflow_plugin/datahub_listener.py +logger.debug("this is the sample debug line") + +# run the DAG again and you can see the debug lines in the task_run log at, +#1. click on the `timestamp` in the `Last Run` column +#2. select the task +#3. click on the `log` option ``` + + +> **P.S. if you are not able to see the log lines, then restart the `airflow scheduler` and rerun the DAG** + ### (Optional) Set up your Python environment for developing on Dagster Plugin From the repository root: diff --git a/metadata-ingestion/docs/dev_guides/classification.md b/metadata-ingestion/docs/dev_guides/classification.md index be7e1e5013318..f20638a2ab5bd 100644 --- a/metadata-ingestion/docs/dev_guides/classification.md +++ b/metadata-ingestion/docs/dev_guides/classification.md @@ -339,7 +339,7 @@ source: regex: [] library: - spacy - Full_name: + Full_Name: prediction_factors_and_weights: name: 0.3 description: 0 @@ -452,4 +452,4 @@ source: ### DataHub Blog -* [PII Classification just got easier with DataHub](https://blog.datahubproject.io/pii-classification-just-got-easier-with-datahub-6bab2b63abcb) \ No newline at end of file +* [PII Classification just got easier with DataHub](https://blog.datahubproject.io/pii-classification-just-got-easier-with-datahub-6bab2b63abcb) diff --git a/metadata-ingestion/docs/sources/athena/athena_pre.md b/metadata-ingestion/docs/sources/athena/athena_pre.md index a56457d3f84fc..fdb2b136a5255 100644 --- a/metadata-ingestion/docs/sources/athena/athena_pre.md +++ b/metadata-ingestion/docs/sources/athena/athena_pre.md @@ -35,7 +35,7 @@ In order to execute this source, you will need to create a policy with below per "glue:GetPartitions", "s3:GetObject", "s3:ListBucket", - "s3:GetBucketLocation", + "s3:GetBucketLocation" ], "Resource": [ "arn:aws:athena:${region-id}:${account-id}:datacatalog/*", @@ -64,9 +64,9 @@ In order to execute this source, you will need to create a policy with below per "arn:aws:s3:::${athena-query-result-bucket}/*", "arn:aws:s3:::${athena-query-result-bucket}" ] - }, + } ] } ``` -Replace `${var}` with appropriate values as per your athena setup. \ No newline at end of file +Replace `${var}` with appropriate values as per your athena setup. diff --git a/metadata-ingestion/docs/sources/bigquery/bigquery_pre.md b/metadata-ingestion/docs/sources/bigquery/bigquery_pre.md index 0d856b915629d..d6efe9334f756 100644 --- a/metadata-ingestion/docs/sources/bigquery/bigquery_pre.md +++ b/metadata-ingestion/docs/sources/bigquery/bigquery_pre.md @@ -28,19 +28,21 @@ There are two important concepts to understand and identify: If you have multiple projects in your BigQuery setup, the role should be granted these permissions in each of the projects. ::: -| permission                       | Description                                                                                                 | Capability               | Default GCP role which contains this permission                                                                 | -|----------------------------------|--------------------------------------------------------------------------------------------------------------|-------------------------------------|-----------------------------------------------------------------------------------------------------------------| -| `bigquery.datasets.get`         | Retrieve metadata about a dataset.                                                                           | Table Metadata Extraction           | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | -| `bigquery.datasets.getIamPolicy` | Read a dataset's IAM permissions.                                                                           | Table Metadata Extraction           | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | -| `bigquery.tables.list`           | List BigQuery tables.                                                                                       | Table Metadata Extraction           | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | -| `bigquery.tables.get`           | Retrieve metadata for a table.                                                                               | Table Metadata Extraction           | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | -| `bigquery.routines.get`           | Get Routines. Needs to retrieve metadata for a table from system table.                                                                                       | Table Metadata Extraction           | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | -| `bigquery.routines.list`           | List Routines. Needs to retrieve metadata for a table from system table                                                                               | Table Metadata Extraction           | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | -| `resourcemanager.projects.get`   | Retrieve project names and metadata.                                                                         | Table Metadata Extraction           | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | -| `bigquery.jobs.listAll`         | List all jobs (queries) submitted by any user. Needs for Lineage extraction.                                 | Lineage Extraction/Usage extraction | [roles/bigquery.resourceViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.resourceViewer) | -| `logging.logEntries.list`       | Fetch log entries for lineage/usage data. Not required if `use_exported_bigquery_audit_metadata` is enabled. | Lineage Extraction/Usage extraction | [roles/logging.privateLogViewer](https://cloud.google.com/logging/docs/access-control#logging.privateLogViewer) | -| `logging.privateLogEntries.list` | Fetch log entries for lineage/usage data. Not required if `use_exported_bigquery_audit_metadata` is enabled. | Lineage Extraction/Usage extraction | [roles/logging.privateLogViewer](https://cloud.google.com/logging/docs/access-control#logging.privateLogViewer) | -| `bigquery.tables.getData`       | Access table data to extract storage size, last updated at, data profiles etc. | Profiling                           |                                                                                                                 | +| Permission | Description | Capability | Default GCP Role Which Contains This Permission | +|----------------------------------|-----------------------------------------------------------------------------------------------------------------|-------------------------------------|---------------------------------------------------------------------------| +| `bigquery.datasets.get` | Retrieve metadata about a dataset. | Table Metadata Extraction | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | +| `bigquery.datasets.getIamPolicy` | Read a dataset's IAM permissions. | Table Metadata Extraction | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | +| `bigquery.tables.list` | List BigQuery tables. | Table Metadata Extraction | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | +| `bigquery.tables.get` | Retrieve metadata for a table. | Table Metadata Extraction | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | +| `bigquery.routines.get` | Get Routines. Needs to retrieve metadata for a table from system table. | Table Metadata Extraction | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | +| `bigquery.routines.list` | List Routines. Needs to retrieve metadata for a table from system table. | Table Metadata Extraction | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | +| `resourcemanager.projects.get` | Retrieve project names and metadata. | Table Metadata Extraction | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | +| `bigquery.jobs.listAll` | List all jobs (queries) submitted by any user. Needs for Lineage extraction. | Lineage Extraction/Usage Extraction | [roles/bigquery.resourceViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.resourceViewer) | +| `logging.logEntries.list` | Fetch log entries for lineage/usage data. Not required if `use_exported_bigquery_audit_metadata` is enabled. | Lineage Extraction/Usage Extraction | [roles/logging.privateLogViewer](https://cloud.google.com/logging/docs/access-control#logging.privateLogViewer) | +| `logging.privateLogEntries.list` | Fetch log entries for lineage/usage data. Not required if `use_exported_bigquery_audit_metadata` is enabled. | Lineage Extraction/Usage Extraction | [roles/logging.privateLogViewer](https://cloud.google.com/logging/docs/access-control#logging.privateLogViewer) | +| `bigquery.tables.getData` | Access table data to extract storage size, last updated at, data profiles etc. | Profiling | | +| `datacatalog.policyTags.get` | *Optional* Get policy tags for columns with associated policy tags. This permission is required only if `extract_policy_tags_from_catalog` is enabled. | Policy Tag Extraction | [roles/datacatalog.viewer](https://cloud.google.com/data-catalog/docs/access-control#permissions-and-roles) | + #### Create a service account in the Extractor Project diff --git a/metadata-ingestion/docs/sources/csv/csv-enricher_recipe.yml b/metadata-ingestion/docs/sources/csv-enricher/csv-enricher_recipe.yml similarity index 100% rename from metadata-ingestion/docs/sources/csv/csv-enricher_recipe.yml rename to metadata-ingestion/docs/sources/csv-enricher/csv-enricher_recipe.yml diff --git a/metadata-ingestion/docs/sources/databricks/README.md b/metadata-ingestion/docs/sources/databricks/README.md index 32b0b20c9480b..a6cf39084c6ab 100644 --- a/metadata-ingestion/docs/sources/databricks/README.md +++ b/metadata-ingestion/docs/sources/databricks/README.md @@ -11,7 +11,7 @@ The alternative way to integrate is via the Hive connector. The [Hive starter re ## Databricks Spark -To complete the picture, we recommend adding push-based ingestion from your Spark jobs to see real-time activity and lineage between your Databricks tables and your Spark jobs. Use the Spark agent to push metadata to DataHub using the instructions [here](../../../../metadata-integration/java/spark-lineage/README.md#configuration-instructions-databricks). +To complete the picture, we recommend adding push-based ingestion from your Spark jobs to see real-time activity and lineage between your Databricks tables and your Spark jobs. Use the Spark agent to push metadata to DataHub using the instructions [here](../../../../metadata-integration/java/spark-lineage-beta/README.md#configuration-instructions-databricks). ## Watch the DataHub Talk at the Data and AI Summit 2022 diff --git a/metadata-ingestion/docs/sources/datahub/datahub_recipe.yml b/metadata-ingestion/docs/sources/datahub/datahub_recipe.yml index 2885b00d819f5..9012aa6029611 100644 --- a/metadata-ingestion/docs/sources/datahub/datahub_recipe.yml +++ b/metadata-ingestion/docs/sources/datahub/datahub_recipe.yml @@ -19,14 +19,15 @@ source: enabled: true ignore_old_state: false urn_pattern: - deny: - # Ignores all datahub metadata where the urn matches the regex - - ^denied.urn.* - allow: - # Ingests all datahub metadata where the urn matches the regex. - - ^allowed.urn.* - extractor_config: - set_system_metadata: false # Replicate system metadata + deny: + # Ignores all datahub metadata where the urn matches the regex + - ^denied.urn.* + allow: + # Ingests all datahub metadata where the urn matches the regex. + - ^allowed.urn.* + +flags: + set_system_metadata: false # Replicate system metadata # Here, we write to a DataHub instance # You can also use a different sink, e.g. to write the data to a file instead diff --git a/metadata-ingestion/docs/sources/iceberg/iceberg.md b/metadata-ingestion/docs/sources/iceberg/iceberg.md index 07ab56f113030..7e40315a2e319 100644 --- a/metadata-ingestion/docs/sources/iceberg/iceberg.md +++ b/metadata-ingestion/docs/sources/iceberg/iceberg.md @@ -10,7 +10,7 @@ This ingestion source maps the following Source System Concepts to DataHub Conce | Source Concept | DataHub Concept | Notes | | -- | -- | -- | | `iceberg` | [Data Platform](docs/generated/metamodel/entities/dataPlatform.md) | | -| Table | [Dataset](docs/generated/metamodel/entities/dataset.md) | Each Iceberg table maps to a Dataset named using the parent folders. If a table is stored under `my/namespace/table`, the dataset name will be `my.namespace.table`. If a [Platform Instance](https://datahubproject.io/docs/platform-instances/) is configured, it will be used as a prefix: `.my.namespace.table`. | +| Table | [Dataset](docs/generated/metamodel/entities/dataset.md) | An Iceberg table is registered inside a catalog using a name, where the catalog is responsible for creating, dropping and renaming tables. Catalogs manage a collection of tables that are usually grouped into namespaces. The name of a table is mapped to a Dataset name. If a [Platform Instance](https://datahubproject.io/docs/platform-instances/) is configured, it will be used as a prefix: `.my.namespace.table`. | | [Table property](https://iceberg.apache.org/docs/latest/configuration/#table-properties) | [User (a.k.a CorpUser)](docs/generated/metamodel/entities/corpuser.md) | The value of a table property can be used as the name of a CorpUser owner. This table property name can be configured with the source option `user_ownership_property`. | | [Table property](https://iceberg.apache.org/docs/latest/configuration/#table-properties) | CorpGroup | The value of a table property can be used as the name of a CorpGroup owner. This table property name can be configured with the source option `group_ownership_property`. | | Table parent folders (excluding [warehouse catalog location](https://iceberg.apache.org/docs/latest/configuration/#catalog-properties)) | Container | Available in a future release | diff --git a/metadata-ingestion/docs/sources/iceberg/iceberg_recipe.yml b/metadata-ingestion/docs/sources/iceberg/iceberg_recipe.yml index 8caedafbea50e..c8deb8fcc9340 100644 --- a/metadata-ingestion/docs/sources/iceberg/iceberg_recipe.yml +++ b/metadata-ingestion/docs/sources/iceberg/iceberg_recipe.yml @@ -3,17 +3,25 @@ source: config: env: PROD catalog: - name: my_iceberg_catalog - type: rest - # Catalog configuration follows pyiceberg's documentation (https://py.iceberg.apache.org/configuration) - config: + # REST catalog configuration example using S3 storage + my_rest_catalog: + type: rest + # Catalog configuration follows pyiceberg's documentation (https://py.iceberg.apache.org/configuration) uri: http://localhost:8181 s3.access-key-id: admin s3.secret-access-key: password s3.region: us-east-1 warehouse: s3a://warehouse/wh/ s3.endpoint: http://localhost:9000 - platform_instance: my_iceberg_catalog + # SQL catalog configuration example using Azure datalake storage and a PostgreSQL database + # my_sql_catalog: + # type: sql + # uri: postgresql+psycopg2://user:password@sqldatabase.postgres.database.azure.com:5432/icebergcatalog + # adlfs.tenant-id: + # adlfs.account-name: + # adlfs.client-id: + # adlfs.client-secret: + platform_instance: my_rest_catalog table_pattern: allow: - marketing.* @@ -21,5 +29,4 @@ source: enabled: true sink: - # sink configs - + # sink configs \ No newline at end of file diff --git a/metadata-ingestion/docs/sources/file/file_recipe.yml b/metadata-ingestion/docs/sources/metadata-file/metadata-file_recipe.yml similarity index 100% rename from metadata-ingestion/docs/sources/file/file_recipe.yml rename to metadata-ingestion/docs/sources/metadata-file/metadata-file_recipe.yml diff --git a/metadata-ingestion/docs/sources/oracle/oracle_pre.md b/metadata-ingestion/docs/sources/oracle/oracle_pre.md new file mode 100644 index 0000000000000..aa4cf64efac73 --- /dev/null +++ b/metadata-ingestion/docs/sources/oracle/oracle_pre.md @@ -0,0 +1,24 @@ +### Prerequisites + +#### Data Dictionary Mode/Views + +The Oracle ingestion source supports two modes for extracting metadata information (see `data_dictionary_mode` option): `ALL` and `DBA`. In the `ALL` mode, the SQLAlchemy backend queries `ALL_` data dictionary views to extract metadata information. In the `DBA` mode, the Oracle ingestion source directly queries `DBA_` data dictionary views to extract metadata information. `ALL_` views only provide information accessible to the user used for ingestion while `DBA_` views provide information for the entire database (that is, all schema objects in the database). + +The following table contains a brief description of what each data dictionary view is used for: + +| Data Dictionary View | What's it used for? | +| --- | --- | +| `ALL_TABLES` or `DBA_TABLES` | Get list of all relational tables in the database | +| `ALL_VIEWS` or `DBA_VIEWS` | Get list of all views in the database | +| `ALL_TAB_COMMENTS` or `DBA_TAB_COMMENTS` | Get comments on tables and views | +| `ALL_TAB_COLS` or `DBA_TAB_COLS` | Get description of the columns of tables and views | +| `ALL_COL_COMMENTS` or `DBA_COL_COMMENTS` | Get comments on the columns of tables and views | +| `ALL_TAB_IDENTITY_COLS` or `DBA_TAB_IDENTITY_COLS` | Get table identity columns | +| `ALL_CONSTRAINTS` or `DBA_CONSTRAINTS` | Get constraint definitions on tables | +| `ALL_CONS_COLUMNS` or `DBA_CONS_COLUMNS` | Get list of columns that are specified in constraints | +| `ALL_USERS` or `DBA_USERS` | Get all schema names | + +#### Data Dictionary Views accessible information and required privileges + +- `ALL_` views display all the information accessible to the user used for ingestion, including information from the user's schema as well as information from objects in other schemas, if the user has access to those objects by way of grants of privileges or roles. +- `DBA_` views display all relevant information in the entire database. They can be queried only by users with the `SYSDBA` system privilege or `SELECT ANY DICTIONARY` privilege, or `SELECT_CATALOG_ROLE` role, or by users with direct privileges granted to them. diff --git a/metadata-ingestion/docs/sources/powerbi/powerbi_pre.md b/metadata-ingestion/docs/sources/powerbi/powerbi_pre.md index 52fe1cd77e8fd..b581e5fc8f70d 100644 --- a/metadata-ingestion/docs/sources/powerbi/powerbi_pre.md +++ b/metadata-ingestion/docs/sources/powerbi/powerbi_pre.md @@ -108,6 +108,14 @@ By default, extracting endorsement information to tags is disabled. The feature Please note that the default implementation overwrites tags for the ingested entities, if you need to preserve existing tags, consider using a [transformer](../../../../metadata-ingestion/docs/transformer/dataset_transformer.md#simple-add-dataset-globaltags) with `semantics: PATCH` tags instead of `OVERWRITE`. +## Profiling + +The profiling implementation is done through querying [DAX query endpoint](https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/execute-queries). Therefore the principal needs to have permission to query the datasets to be profiled. Usually this means that the service principal should have `Contributor` role for the workspace to be ingested. Profiling is done with column based queries to be able to handle wide datasets without timeouts. + +Take into account that the profiling implementation exeutes fairly big amount of DAX queries and for big datasets this is substantial load to the PowerBI system. + +The `profiling_pattern` setting may be used to limit profiling actions to only a certain set of resources in PowerBI. Both allow and deny rules are matched against following pattern for every table in a PowerBI Dataset: `workspace_name.dataset_name.table_name`. User may limit profiling with these settings at table level, dataset level or workspace level. + ## Admin Ingestion vs. Basic Ingestion PowerBI provides two sets of API i.e. [Basic API and Admin API](https://learn.microsoft.com/en-us/rest/api/power-bi/). @@ -140,6 +148,7 @@ If you don't want to add a service principal as a member in your workspace, then Caveats of setting `admin_apis_only` to `true`: - Report's pages would not get ingested as page API is not available in PowerBI Admin API - [PowerBI Parameters](https://learn.microsoft.com/en-us/power-query/power-query-query-parameters) would not get resolved to actual values while processing M-Query for table lineage + - Dataset profiling is unavailable, as it requires access to the workspace API ### Basic Ingestion: Service Principal As Member In Workspace diff --git a/metadata-ingestion/docs/sources/powerbi/powerbi_recipe.yml b/metadata-ingestion/docs/sources/powerbi/powerbi_recipe.yml index 31eed0bddaa6a..ebd3dd50cfebc 100644 --- a/metadata-ingestion/docs/sources/powerbi/powerbi_recipe.yml +++ b/metadata-ingestion/docs/sources/powerbi/powerbi_recipe.yml @@ -64,6 +64,16 @@ source: # extract powerbi dataset table schema extract_dataset_schema: true + # Enable PowerBI dataset profiling + profiling: + enabled: false + # Pattern to limit which resources to profile + # Matched resource format is following: + # workspace_name.dataset_name.table_name + profile_pattern: + deny: + - .* + sink: # sink configs diff --git a/metadata-ingestion/docs/sources/s3/README.md b/metadata-ingestion/docs/sources/s3/README.md index 7944f78280a42..b0d354a9b3c2a 100644 --- a/metadata-ingestion/docs/sources/s3/README.md +++ b/metadata-ingestion/docs/sources/s3/README.md @@ -1,19 +1,11 @@ -This connector ingests S3 datasets into DataHub. It allows mapping an individual file or a folder of files to a dataset in DataHub. +This connector ingests AWS S3 datasets into DataHub. It allows mapping an individual file or a folder of files to a dataset in DataHub. To specify the group of files that form a dataset, use `path_specs` configuration in ingestion recipe. Refer section [Path Specs](https://datahubproject.io/docs/generated/ingestion/sources/s3/#path-specs) for more details. -### Concept Mapping - -This ingestion source maps the following Source System Concepts to DataHub Concepts: - -| Source Concept | DataHub Concept | Notes | -| ---------------------------------------- |--------------------------------------------------------------------------------------------| ------------------- | -| `"s3"` | [Data Platform](https://datahubproject.io/docs/generated/metamodel/entities/dataplatform/) | | -| s3 object / Folder containing s3 objects | [Dataset](https://datahubproject.io/docs/generated/metamodel/entities/dataset/) | | -| s3 bucket | [Container](https://datahubproject.io/docs/generated/metamodel/entities/container/) | Subtype `S3 bucket` | -| s3 folder | [Container](https://datahubproject.io/docs/generated/metamodel/entities/container/) | Subtype `Folder` | +:::tip +This connector can also be used to ingest local files. +Just replace `s3://` in your path_specs with an absolute path to files on the machine running ingestion. +::: -This connector supports both local files as well as those stored on AWS S3 (which must be identified using the prefix `s3://`). -[a] ### Supported file types Supported file types are as follows: @@ -30,6 +22,16 @@ Schemas for schemaless formats (CSV, TSV, JSONL, JSON) are inferred. For CSV, TS JSON file schemas are inferred on the basis of the entire file (given the difficulty in extracting only the first few objects of the file), which may impact performance. We are working on using iterator-based JSON parsers to avoid reading in the entire JSON object. +### Concept Mapping + +This ingestion source maps the following Source System Concepts to DataHub Concepts: + +| Source Concept | DataHub Concept | Notes | +| ---------------------------------------- |--------------------------------------------------------------------------------------------| ------------------- | +| `"s3"` | [Data Platform](https://datahubproject.io/docs/generated/metamodel/entities/dataplatform/) | | +| s3 object / Folder containing s3 objects | [Dataset](https://datahubproject.io/docs/generated/metamodel/entities/dataset/) | | +| s3 bucket | [Container](https://datahubproject.io/docs/generated/metamodel/entities/container/) | Subtype `S3 bucket` | +| s3 folder | [Container](https://datahubproject.io/docs/generated/metamodel/entities/container/) | Subtype `Folder` | ### Profiling @@ -42,4 +44,4 @@ This plugin extracts: - histograms or frequencies of unique values Note that because the profiling is run with PySpark, we require Spark 3.0.3 with Hadoop 3.2 to be installed (see [compatibility](#compatibility) for more details). If profiling, make sure that permissions for **s3a://** access are set because Spark and Hadoop use the s3a:// protocol to interface with AWS (schema inference outside of profiling requires s3:// access). -Enabling profiling will slow down ingestion runs. \ No newline at end of file +Enabling profiling will slow down ingestion runs. diff --git a/metadata-ingestion/docs/sources/s3/s3_recipe.yml b/metadata-ingestion/docs/sources/s3/s3_recipe.yml index 693b9528373ab..301e811b76926 100644 --- a/metadata-ingestion/docs/sources/s3/s3_recipe.yml +++ b/metadata-ingestion/docs/sources/s3/s3_recipe.yml @@ -1,9 +1,9 @@ +# Ingest data from S3 source: type: s3 config: path_specs: - - - include: "s3://covid19-lake/covid_knowledge_graph/csv/nodes/*.*" + - include: "s3://covid19-lake/covid_knowledge_graph/csv/nodes/*.*" aws_config: aws_access_key_id: ***** @@ -13,4 +13,9 @@ source: profiling: enabled: false -# sink configs +# Ingest data from local filesystem +source: + type: s3 + config: + path_specs: + - include: "/absolute/path/*.csv" diff --git a/metadata-ingestion/docs/transformer/dataset_transformer.md b/metadata-ingestion/docs/transformer/dataset_transformer.md index 64d1438cfcc73..773a7e8554832 100644 --- a/metadata-ingestion/docs/transformer/dataset_transformer.md +++ b/metadata-ingestion/docs/transformer/dataset_transformer.md @@ -10,7 +10,7 @@ The below table shows transformer which can transform aspects of entity [Dataset | `ownership` | - [Simple Add Dataset ownership](#simple-add-dataset-ownership)
- [Pattern Add Dataset ownership](#pattern-add-dataset-ownership)
- [Simple Remove Dataset Ownership](#simple-remove-dataset-ownership)
- [Extract Ownership from Tags](#extract-ownership-from-tags)
- [Clean suffix prefix from Ownership](#clean-suffix-prefix-from-ownership) | | `globalTags` | - [Simple Add Dataset globalTags ](#simple-add-dataset-globaltags)
- [Pattern Add Dataset globalTags](#pattern-add-dataset-globaltags)
- [Add Dataset globalTags](#add-dataset-globaltags) | | `browsePaths` | - [Set Dataset browsePath](#set-dataset-browsepath) | -| `glossaryTerms` | - [Simple Add Dataset glossaryTerms ](#simple-add-dataset-glossaryterms)
- [Pattern Add Dataset glossaryTerms](#pattern-add-dataset-glossaryterms) | +| `glossaryTerms` | - [Simple Add Dataset glossaryTerms ](#simple-add-dataset-glossaryterms)
- [Pattern Add Dataset glossaryTerms](#pattern-add-dataset-glossaryterms)
- [Tags to Term Mapping](#tags-to-term-mapping) | | `schemaMetadata` | - [Pattern Add Dataset Schema Field glossaryTerms](#pattern-add-dataset-schema-field-glossaryterms)
- [Pattern Add Dataset Schema Field globalTags](#pattern-add-dataset-schema-field-globaltags) | | `datasetProperties` | - [Simple Add Dataset datasetProperties](#simple-add-dataset-datasetproperties)
- [Add Dataset datasetProperties](#add-dataset-datasetproperties) | | `domains` | - [Simple Add Dataset domains](#simple-add-dataset-domains)
- [Pattern Add Dataset domains](#pattern-add-dataset-domains)
- [Domain Mapping Based on Tags](#domain-mapping-based-on-tags) | @@ -126,7 +126,7 @@ transformers: |--------------------|----------|--------------|-------------|---------------------------------------------------------------------| | `owner_urns` | ✅ | list[string] | | List of owner urns. | | `ownership_type` | | string | "DATAOWNER" | ownership type of the owners (either as enum or ownership type urn) | -| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `replace_existing` | | boolean | `false` | Whether to remove ownership from entity sent by ingestion source. | | `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | For transformer behaviour on `replace_existing` and `semantics`, please refer section [Relationship Between replace_existing And semantics](#relationship-between-replace_existing-and-semantics). @@ -270,7 +270,7 @@ Note that whatever owners you send via `simple_remove_dataset_ownership` will ov |-----------------------------|----------|--------------|---------------|------------------------------------------------------------------| | `extract_tags_from` | ✅ | string | `urn` | Which field to extract tag from. Currently only `urn` is supported. | | `extract_tags_regex` | ✅ | string | `.*` | Regex to use to extract tag.| -| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `replace_existing` | | boolean | `false` | Whether to remove globalTags from entity sent by ingestion source. | | `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | Let’s suppose we’d like to add a dataset tags based on part of urn. To do so, we can use the `extract_dataset_tags` transformer that’s included in the ingestion framework. @@ -297,7 +297,7 @@ a tag called `USA-ops-team` and `Canada-marketing` will be added to them respect | Field | Required | Type | Default | Description | |-----------------------------|----------|--------------|---------------|------------------------------------------------------------------| | `tag_urns` | ✅ | list[string] | | List of globalTags urn. | -| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `replace_existing` | | boolean | `false` | Whether to remove globalTags from entity sent by ingestion source. | | `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | Let’s suppose we’d like to add a set of dataset tags. To do so, we can use the `simple_add_dataset_tags` transformer that’s included in the ingestion framework. @@ -350,7 +350,7 @@ The config, which we’d append to our ingestion recipe YAML, would look like th | Field | Required | Type | Default | Description | |-----------------------------|----------|----------------------|-------------|---------------------------------------------------------------------------------------| | `tag_pattern` | ✅ | map[regx, list[urn]] | | Entity urn with regular expression and list of tags urn apply to matching entity urn. | -| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `replace_existing` | | boolean | `false` | Whether to remove globalTags from entity sent by ingestion source. | | `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | Let’s suppose we’d like to append a series of tags to specific datasets. To do so, we can use the `pattern_add_dataset_tags` module that’s included in the ingestion framework. This will match the regex pattern to `urn` of the dataset and assign the respective tags urns given in the array. @@ -407,7 +407,7 @@ The config, which we’d append to our ingestion recipe YAML, would look like th | Field | Required | Type | Default | Description | |-----------------------------|----------|--------------------------------------------|---------------|----------------------------------------------------------------------------| | `get_tags_to_add` | ✅ | callable[[str], list[TagAssociationClass]] | | A function which takes entity urn as input and return TagAssociationClass. | -| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `replace_existing` | | boolean | `false` | Whether to remove globalTags from entity sent by ingestion source. | | `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | If you'd like to add more complex logic for assigning tags, you can use the more generic add_dataset_tags transformer, which calls a user-provided function to determine the tags for each dataset. @@ -477,7 +477,7 @@ Finally, you can install and use your custom transformer as [shown here](#instal | Field | Required | Type | Default | Description | |-----------------------------|----------|--------------|--------------|------------------------------------------------------------------| | `path_templates` | ✅ | list[string] | | List of path templates. | -| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `replace_existing` | | boolean | `false` | Whether to remove browsePath from entity sent by ingestion source. | | `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | If you would like to add to browse paths of dataset can use this transformer. There are 3 optional variables that you can use to get information from the dataset `urn`: @@ -562,7 +562,7 @@ In this case, the resulting dataset will have only 1 browse path, the one from t | Field | Required | Type | Default | Description | |-----------------------------|----------|--------------|---------------|------------------------------------------------------------------| | `term_urns` | ✅ | list[string] | | List of glossaryTerms urn. | -| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `replace_existing` | | boolean | `false` | Whether to remove glossaryTerms from entity sent by ingestion source. | | `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | We can use a similar convention to associate [Glossary Terms](../../../docs/generated/ingestion/sources/business-glossary.md) to datasets. @@ -617,7 +617,7 @@ The config, which we’d append to our ingestion recipe YAML, would look like th | Field | Required | Type | Default | Description | |-----------------------------|--------|----------------------|--------------|-------------------------------------------------------------------------------------------------| | `term_pattern` | ✅ | map[regx, list[urn]] | | entity urn with regular expression and list of glossaryTerms urn apply to matching entity urn. | -| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `replace_existing` | | boolean | `false` | Whether to remove glossaryTerms from entity sent by ingestion source. | | `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | We can add glossary terms to datasets based on a regex filter. @@ -668,12 +668,63 @@ We can add glossary terms to datasets based on a regex filter. ".*example1.*": ["urn:li:glossaryTerm:Email", "urn:li:glossaryTerm:Address"] ".*example2.*": ["urn:li:glossaryTerm:PostalCode"] ``` + +## Tags to Term Mapping +### Config Details + +| Field | Required | Type | Default | Description | +|---------------|----------|--------------------|-------------|-------------------------------------------------------------------------------------------------------| +| `tags` | ✅ | List[str] | | List of tag names based on which terms will be created and associated with the dataset. | +| `semantics` | | enum | "OVERWRITE" | Determines whether to OVERWRITE or PATCH the terms associated with the dataset on DataHub GMS. | + +
+ +The `tags_to_term` transformer is designed to map specific tags to glossary terms within DataHub. It takes a configuration of tags that should be translated into corresponding glossary terms. This transformer can apply these mappings to any tags found either at the column level of a dataset or at the dataset top level. + +When specifying tags in the configuration, use the tag's simple name rather than the full tag URN. + +For example, instead of using the tag URN `urn:li:tag:snowflakedb.snowflakeschema.tag_name:tag_value`, you should specify just the tag name `tag_name` in the mapping configuration. + +```yaml +transformers: + - type: "tags_to_term" + config: + semantics: OVERWRITE # OVERWRITE is the default behavior + tags: + - "tag_name" +``` + +The `tags_to_term` transformer can be configured in the following ways: + +- Add terms based on tags, however overwrite the terms available for the dataset on DataHub GMS +```yaml + transformers: + - type: "tags_to_term" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + tags: + - "example1" + - "example2" + - "example3" + ``` +- Add terms based on tags, however keep the terms available for the dataset on DataHub GMS +```yaml + transformers: + - type: "tags_to_term" + config: + semantics: PATCH + tags: + - "example1" + - "example2" + - "example3" + ``` + ## Pattern Add Dataset Schema Field glossaryTerms ### Config Details | Field | Required | Type | Default | Description | |-----------------------------|---------|----------------------|-------------|------------------------------------------------------------------------------------------------| | `term_pattern` | ✅ | map[regx, list[urn]] | | entity urn with regular expression and list of glossaryTerms urn apply to matching entity urn. | -| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `replace_existing` | | boolean | `false` | Whether to remove glossaryTerms from entity sent by ingestion source. | | `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | We can add glossary terms to schema fields based on a regex filter. @@ -730,7 +781,7 @@ Note that only terms from the first matching pattern will be applied. | Field | Required | Type | Default | Description | |-----------------------------|----------|----------------------|-------------|---------------------------------------------------------------------------------------| | `tag_pattern` | ✅ | map[regx, list[urn]] | | entity urn with regular expression and list of tags urn apply to matching entity urn. | -| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `replace_existing` | | boolean | `false` | Whether to remove globalTags from entity sent by ingestion source. | | `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | @@ -790,7 +841,7 @@ The config would look like this: | Field | Required | Type | Default | Description | |--------------------|---------|----------------|-------------|------------------------------------------------------------------| | `properties` | ✅ | dict[str, str] | | Map of key value pair. | -| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `replace_existing` | | boolean | `false` | Whether to remove datasetProperties from entity sent by ingestion source. | | `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | `simple_add_dataset_properties` transformer assigns the properties to dataset entity from the configuration. @@ -817,8 +868,6 @@ overwrite the previous value. properties: prop1: value1 prop2: value2 - - ``` - Add dataset-properties, however overwrite the dataset-properties available for the dataset on DataHub GMS ```yaml @@ -829,8 +878,6 @@ overwrite the previous value. properties: prop1: value1 prop2: value2 - - ``` - Add dataset-properties, however keep the dataset-properties available for the dataset on DataHub GMS ```yaml @@ -841,7 +888,6 @@ overwrite the previous value. properties: prop1: value1 prop2: value2 - ``` ## Add Dataset datasetProperties @@ -849,7 +895,7 @@ overwrite the previous value. | Field | Required | Type | Default | Description | |--------------------------------|----------|--------------------------------------------|-------------|------------------------------------------------------------------| | `add_properties_resolver_class`| ✅ | Type[AddDatasetPropertiesResolverBase] | | A class extends from `AddDatasetPropertiesResolverBase` | -| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `replace_existing` | | boolean | `false` | Whether to remove datasetProperties from entity sent by ingestion source. | | `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | If you'd like to add more complex logic for assigning properties, you can use the `add_dataset_properties` transformer, which calls a user-provided class (that extends from `AddDatasetPropertiesResolverBase` class) to determine the properties for each dataset. @@ -948,7 +994,7 @@ transformers: | Field | Required | Type | Default | Description | |--------------------|----------|------------------------|---------------|------------------------------------------------------------------| | `domains` | ✅ | list[union[urn, str]] | | List of simple domain name or domain urns. | -| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `replace_existing` | | boolean | `false` | Whether to remove domains from entity sent by ingestion source. | | `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | For transformer behaviour on `replace_existing` and `semantics`, please refer section [Relationship Between replace_existing And semantics](#relationship-between-replace_existing-and-semantics). @@ -973,7 +1019,7 @@ transformers: `simple_add_dataset_domain` can be configured in below different way - Add domains, however replace existing domains sent by ingestion source -```yaml + ```yaml transformers: - type: "simple_add_dataset_domain" config: @@ -981,9 +1027,9 @@ transformers: domains: - "urn:li:domain:engineering" - "urn:li:domain:hr" - ``` + ``` - Add domains, however overwrite the domains available for the dataset on DataHub GMS -```yaml + ```yaml transformers: - type: "simple_add_dataset_domain" config: @@ -991,9 +1037,9 @@ transformers: domains: - "urn:li:domain:engineering" - "urn:li:domain:hr" - ``` + ``` - Add domains, however keep the domains available for the dataset on DataHub GMS -```yaml + ```yaml transformers: - type: "simple_add_dataset_domain" config: @@ -1001,14 +1047,14 @@ transformers: domains: - "urn:li:domain:engineering" - "urn:li:domain:hr" - ``` + ``` ## Pattern Add Dataset domains ### Config Details | Field | Required | Type | Default | Description | |----------------------------|-----------|---------------------------------|-----------------|----------------------------------------------------------------------------------------------------------------------------| | `domain_pattern` | ✅ | map[regx, list[union[urn, str]] | | dataset urn with regular expression and list of simple domain name or domain urn need to be apply on matching dataset urn. | -| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `replace_existing` | | boolean | `false` | Whether to remove domains from entity sent by ingestion source. | | `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | Let’s suppose we’d like to append a series of domain to specific datasets. To do so, we can use the pattern_add_dataset_domain transformer that’s included in the ingestion framework. @@ -1019,20 +1065,20 @@ Here we can set domain list to either urn (i.e. urn:li:domain:hr) or simple doma in both of the cases domain should be provisioned on DataHub GMS ```yaml - transformers: - - type: "pattern_add_dataset_domain" - config: - semantics: OVERWRITE - domain_pattern: - rules: - 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.n.*': ["hr"] - 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.t.*': ["urn:li:domain:finance"] + transformers: + - type: "pattern_add_dataset_domain" + config: + semantics: OVERWRITE + domain_pattern: + rules: + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.n.*': ["hr"] + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.t.*': ["urn:li:domain:finance"] ``` `pattern_add_dataset_domain` can be configured in below different way - Add domains, however replace existing domains sent by ingestion source -```yaml + ```yaml transformers: - type: "pattern_add_dataset_domain" config: @@ -1041,29 +1087,29 @@ in both of the cases domain should be provisioned on DataHub GMS rules: 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.n.*': ["hr"] 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.t.*': ["urn:li:domain:finance"] - ``` + ``` - Add domains, however overwrite the domains available for the dataset on DataHub GMS -```yaml - transformers: - - type: "pattern_add_dataset_domain" - config: - semantics: OVERWRITE # OVERWRITE is default behaviour - domain_pattern: - rules: - 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.n.*': ["hr"] - 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.t.*': ["urn:li:domain:finance"] - ``` + ```yaml + transformers: + - type: "pattern_add_dataset_domain" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + domain_pattern: + rules: + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.n.*': ["hr"] + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.t.*': ["urn:li:domain:finance"] + ``` - Add domains, however keep the domains available for the dataset on DataHub GMS -```yaml - transformers: - - type: "pattern_add_dataset_domain" - config: - semantics: PATCH - domain_pattern: - rules: - 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.n.*': ["hr"] - 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.t.*': ["urn:li:domain:finance"] - ``` + ```yaml + transformers: + - type: "pattern_add_dataset_domain" + config: + semantics: PATCH + domain_pattern: + rules: + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.n.*': ["hr"] + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.t.*': ["urn:li:domain:finance"] + ``` @@ -1099,7 +1145,7 @@ transformers: `domain_mapping_based_on_tags` can be configured in below different way - Add domains based on tags, however overwrite the domains available for the dataset on DataHub GMS -```yaml + ```yaml transformers: - type: "domain_mapping_based_on_tags" config: @@ -1107,9 +1153,9 @@ transformers: domain_mapping: 'example1': "urn:li:domain:engineering" 'example2': "urn:li:domain:hr" - ``` + ``` - Add domains based on tags, however keep the domains available for the dataset on DataHub GMS -```yaml + ```yaml transformers: - type: "domain_mapping_based_on_tags" config: @@ -1117,7 +1163,7 @@ transformers: domain_mapping: 'example1': "urn:li:domain:engineering" 'example2': "urn:li:domain:hr" - ``` + ``` ## Simple Add Dataset dataProduct ### Config Details @@ -1313,18 +1359,18 @@ Let's begin by adding a `create()` method for parsing our configuration dictiona @classmethod def create(cls, config_dict: dict, ctx: PipelineContext) -> "AddCustomOwnership": - config = AddCustomOwnershipConfig.parse_obj(config_dict) - return cls(config, ctx) + config = AddCustomOwnershipConfig.parse_obj(config_dict) + return cls(config, ctx) ``` Next we need to tell the helper classes which entity types and aspect we are interested in transforming. In this case, we want to only process `dataset` entities and transform the `ownership` aspect. ```python def entity_types(self) -> List[str]: - return ["dataset"] + return ["dataset"] - def aspect_name(self) -> str: - return "ownership" +def aspect_name(self) -> str: + return "ownership" ``` Finally we need to implement the `transform_aspect()` method that does the work of adding our custom ownership classes. This method will be called be the framework with an optional aspect value filled out if the upstream source produced a value for this aspect. The framework takes care of pre-processing both MCE-s and MCP-s so that the `transform_aspect()` function is only called one per entity. Our job is merely to inspect the incoming aspect (or absence) and produce a transformed value for this aspect. Returning `None` from this method will effectively suppress this aspect from being emitted. @@ -1332,24 +1378,24 @@ Finally we need to implement the `transform_aspect()` method that does the work ```python # add this as a function of AddCustomOwnership - def transform_aspect( # type: ignore - self, entity_urn: str, aspect_name: str, aspect: Optional[OwnershipClass] - ) -> Optional[OwnershipClass]: +def transform_aspect( # type: ignore + self, entity_urn: str, aspect_name: str, aspect: Optional[OwnershipClass] +) -> Optional[OwnershipClass]: - owners_to_add = self.owners - assert aspect is None or isinstance(aspect, OwnershipClass) + owners_to_add = self.owners + assert aspect is None or isinstance(aspect, OwnershipClass) - if owners_to_add: - ownership = ( - aspect - if aspect - else OwnershipClass( - owners=[], - ) - ) - ownership.owners.extend(owners_to_add) + if owners_to_add: + ownership = ( + aspect + if aspect + else OwnershipClass( + owners=[], + ) + ) + ownership.owners.extend(owners_to_add) - return ownership + return ownership ``` ### More Sophistication: Making calls to DataHub during Transformation @@ -1383,27 +1429,27 @@ e.g. Here is how the AddDatasetOwnership transformer can now support PATCH seman ```python def transform_one(self, mce: MetadataChangeEventClass) -> MetadataChangeEventClass: - if not isinstance(mce.proposedSnapshot, DatasetSnapshotClass): - return mce - owners_to_add = self.config.get_owners_to_add(mce.proposedSnapshot) - if owners_to_add: - ownership = builder.get_or_add_aspect( - mce, - OwnershipClass( - owners=[], - ), - ) - ownership.owners.extend(owners_to_add) - - if self.config.semantics == Semantics.PATCH: - assert self.ctx.graph - patch_ownership = AddDatasetOwnership.get_ownership_to_set( - self.ctx.graph, mce.proposedSnapshot.urn, ownership - ) - builder.set_aspect( - mce, aspect=patch_ownership, aspect_type=OwnershipClass - ) + if not isinstance(mce.proposedSnapshot, DatasetSnapshotClass): return mce + owners_to_add = self.config.get_owners_to_add(mce.proposedSnapshot) + if owners_to_add: + ownership = builder.get_or_add_aspect( + mce, + OwnershipClass( + owners=[], + ), + ) + ownership.owners.extend(owners_to_add) + + if self.config.semantics == Semantics.PATCH: + assert self.ctx.graph + patch_ownership = AddDatasetOwnership.get_ownership_to_set( + self.ctx.graph, mce.proposedSnapshot.urn, ownership + ) + builder.set_aspect( + mce, aspect=patch_ownership, aspect_type=OwnershipClass + ) + return mce ``` ### Installing the package diff --git a/metadata-ingestion/examples/bootstrap_data/banking_business_glossary.yaml b/metadata-ingestion/examples/bootstrap_data/banking_business_glossary.yaml new file mode 100644 index 0000000000000..d0fea81748da5 --- /dev/null +++ b/metadata-ingestion/examples/bootstrap_data/banking_business_glossary.yaml @@ -0,0 +1,71 @@ +version: 1 +source: DataHub +owners: + users: + - mjames +url: "https://github.com/datahub-project/datahub/" +nodes: + - name: FinTech Business Glossary + description: "FinTech Business Glossary" + nodes: + - name: Account + description: "A record of financial transactions and balances associated with a specific customer or entity." + terms: + - name: Savings Account + description: "A type of deposit account that allows customers to deposit and withdraw funds while earning interest on their balance." + custom_properties: + scope: banking + inherits: + - Banking.Deposit + - Banking.Withdrawal + - name: Checking Account + description: "A type of deposit account that enables customers to deposit and withdraw funds, often used for everyday transactions." + domain: "urn:li:domain:banking" + custom_properties: + scope: banking + inherits: + - Banking.Deposit + - Banking.Withdrawal + - name: Regulatory and Compliance + description: "Regulatory and compliance" + owners: + groups: + - complianceOrg + knowledge_links: + - label: Definition for Regulatory and Compliance + url: "https://www.metricstream.com/learn/comprehensive-guide-to-regulatory-compliance.htm" + terms: + - name: Compliance + description: "The adherence to laws, regulations, and internal policies governing the operations of financial institutions." + domain: "urn:li:domain:compliance" + - name: KYC (Know Your Customer) + description: "The process of verifying the identity of customers to prevent fraud." + domain: "urn:li:domain:compliance" + - name: AML (Anti-Money Laundering) + description: "Regulations and procedures designed to detect and prevent the illegal movement of money through financial systems." + domain: "urn:li:domain:compliance" + - name: Banking + description: "Banking" + owners: + users: + - jsmith + terms: + - name: Deposit + description: "The act of placing money into a bank account." + - name: Withdrawal + description: "The removal of funds from a bank account, either by cash withdrawal or electronic transfer." + - name: Transfer + description: "The movement of funds from one account to another, either within the same financial institution or between different institutions." + - name: Overdraft + description: "A negative balance in a bank account resulting from withdrawing more money than is available, often subject to overdraft fees." + - name: Loan + description: "A sum of money borrowed by an individual or entity from a lender, typically repaid with interest over a specified period." + terms: + - name: Mortgage + description: "A type of loan used to finance the purchase of real estate, with the property serving as collateral for the loan." + - name: Personal Loan + description: "A type of loan issued to individuals for various personal expenses, such as debt consolidation, home improvements, or medical bills." + - name: Auto Loan + description: "A type of loan used to finance the purchase of a vehicle, with the vehicle serving as collateral for the loan." + - name: Interest Rate + description: "The rate at which interest is charged on a loan or paid on an investment, expressed as a percentage of the principal amount." \ No newline at end of file diff --git a/metadata-ingestion/examples/bootstrap_data/ecommerce_business_glossary.yaml b/metadata-ingestion/examples/bootstrap_data/ecommerce_business_glossary.yaml new file mode 100644 index 0000000000000..399c925290a83 --- /dev/null +++ b/metadata-ingestion/examples/bootstrap_data/ecommerce_business_glossary.yaml @@ -0,0 +1,112 @@ +version: 1 +source: DataHub +owners: + users: + - mjames +url: "https://github.com/datahub-project/datahub/" +nodes: + - name: E-Commerce Business Glossary + description: "E-Commerce Business Glossary" + nodes: + - name: Customer + description: "A person or entity who purchases products or services from the e-commerce platform." + terms: + - name: User + description: "An individual who interacts with the e-commerce website or application, including both customers and visitors." + inherits: + - Classification.Confidential + custom_properties: + is_confidential: true + - name: Account + description: "A registered user's profile on the e-commerce platform, which includes personal information, order history, and preferences." + domain: Customer Success + inherits: + - Classification.Confidential + custom_properties: + is_confidential: true + - name: Product + description: "Items or services available for sale on the e-commerce platform." + owners: + users: + - mjames + terms: + - name: SKU + description: "A unique identifier assigned to each product to track inventory and sales." + owners: + groups: + - categoryManagement + - name: Inventory + description: "The quantity of products available for sale." + - name: Product Description + description: "Information about a product, including specifications, features, and usage instructions." + - name: Order + description: "A request made by a customer to purchase one or more products." + owners: + users: + - njones + groups: + - logistics + terms: + - name: Cart + description: "A virtual container where customers can add products before proceeding to checkout." + contains: + - Shipping.ZipCode + - Shipping.CountryCode + - Shipping.StreetAddress + domain: "urn:li:domain:Logistics" + - name: Checkout + description: "The process of finalizing and paying for an order." + - name: Payment + description: "The transfer of funds from the customer to the e-commerce company for a purchase." + inherits: + - Classification.HighlyConfidential + - name: Shipping + description: "The process of delivering purchased products to customers." + knowledge_links: + - label: Wiki link for shipping + url: "https://en.wikipedia.org/wiki/Freight_transport" + terms: + - name: ZipCode + description: "Zip code for order" + domain: "urn:li:domain:Logistics" + - name: CountryCode + description: "Country code for order" + domain: "urn:li:domain:Logistics" + - name: StreetAddress + description: "Street address for order" + domain: "urn:li:domain:Logistics" + custom_properties: + is_used_for_compliance_tracking: true + knowledge_links: + - url: "https://en.wikipedia.org/wiki/Address" + label: Wiki link + - name: Privacy Regulations + description: "Standards and best practices for security." + owners: + groups: + - compliance + terms: + - name: GDPR + description: "European Union regulations that govern the collection, use, and protection of personal data." + term_source: "EXTERNAL" + source_ref: FIBO + source_url: "https://gdpr-info.eu/" + - name: CCPA + description: "California Consumer Privacy Act" + term_source: "EXTERNAL" + source_ref: FIBO + source_url: "https://oag.ca.gov/privacy/ccpa" + - name: Privacy Policy + description: "A statement outlining how the e-commerce company collects, uses, and protects customer information." + term_source: "EXTERNAL" + source_ref: FIBO + source_url: "https://en.wikipedia.org/wiki/Privacy_policy" + - name: Classification + description: "A set of terms related to Data Classification" + terms: + - name: Confidential + description: "Confidential Data" + - name: HighlyConfidential + description: "Highly Confidential Data" + - name: Sensitive + description: "Sensitive Data" diff --git a/metadata-ingestion/examples/bootstrap_data/healthcare_business_glossary.yaml b/metadata-ingestion/examples/bootstrap_data/healthcare_business_glossary.yaml new file mode 100644 index 0000000000000..15a6f1b214408 --- /dev/null +++ b/metadata-ingestion/examples/bootstrap_data/healthcare_business_glossary.yaml @@ -0,0 +1,51 @@ +version: 1 +source: DataHub +url: "https://github.com/datahub-project/datahub/" +nodes: + - name: Patient + description: "An individual receiving medical treatment or care from a healthcare provider." + terms: + - name: Age + description: "The age of the patient." + inherits: + - Classification.HighlyConfidential + - name: Gender + description: "The gender of the patient." + inherits: + - Classification.HighlyConfidential + - name: Name + description: "The name of the patient." + inherits: + - Classification.Confidential + - name: HealthcareProvider + description: "An individual or organization that delivers healthcare services, including hospitals, clinics, physicians, nurses, and other medical professionals." + terms: + - name: Primary Care Physician (PCP) + description: "A healthcare provider who serves as a patient's main point of contact for medical care, including preventive services, diagnosis, and treatment of common illnesses." + - name: Specialist + description: "A healthcare provider who focuses on a specific area of medicine or healthcare, such as cardiology, dermatology, or orthopedics." + - name: Nurse Practitioner (NP) + description: "A registered nurse with advanced education and training, authorized to perform physical exams, order diagnostic tests, and prescribe medications." + - name: Billing + description: "The process of submitting and following up on claims with health insurance companies to receive payment for healthcare services provided to patients." + terms: + - name: Medical Coding + description: "The transformation of healthcare diagnoses, procedures, medical services, and equipment into universal medical alphanumeric codes for billing and reporting purposes." + domain: "urn:li:domain:billing" + - name: Health Information Exchange (HIE) + description: "The electronic sharing of health-related information among healthcare organizations, allowing secure access to patient data across different systems and providers." + domain: "urn:li:domain:billing" + - name: Patient Portal + description: "An online platform that allows patients to access their medical records, schedule appointments, communicate with healthcare providers, and manage their healthcare information securely." + domain: "urn:li:domain:billing" + contains: + - Patient.Name + - name: Classification + description: "A set of terms related to Data Classification" + terms: + - name: Confidential + description: "Confidential Data" + - name: HighlyConfidential + description: "Highly Confidential Data" + - name: Sensitive + description: "Sensitive Data" diff --git a/metadata-ingestion/examples/cli_usage/dataset/dataset.yaml b/metadata-ingestion/examples/cli_usage/dataset/dataset.yaml new file mode 100644 index 0000000000000..e4643b3188cc9 --- /dev/null +++ b/metadata-ingestion/examples/cli_usage/dataset/dataset.yaml @@ -0,0 +1,14 @@ +urn: urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.actuating,PROD) +name: ACTUATING +schema_metadata: + fields: + - id: pk + urn: urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.actuating,PROD),pk) + type: "SchemaFieldDataTypeClass({'type': NumberTypeClass({})})" + nativeDataType: NUMBER(1,0) + nullable: true + recursive: false +properties: {} +subtypes: + - Table + diff --git a/metadata-ingestion/examples/data_product/dataproduct.yaml b/metadata-ingestion/examples/data_product/dataproduct.yaml index 587633e2b2217..baf5bb42020a4 100644 --- a/metadata-ingestion/examples/data_product/dataproduct.yaml +++ b/metadata-ingestion/examples/data_product/dataproduct.yaml @@ -22,6 +22,13 @@ tags: terms: - urn:li:glossaryTerm:ClientsAndAccounts.AccountBalance +institutional_memory: + elements: + - title: URL for campaign + description: |- + Go here to see the campaign. + url: https://example.com/pet_of_the_week + # Custom Properties properties: lifecycle: production diff --git a/metadata-ingestion/examples/library/assertions_configuration.yml b/metadata-ingestion/examples/library/assertions_configuration.yml new file mode 100644 index 0000000000000..a44945a30f9a3 --- /dev/null +++ b/metadata-ingestion/examples/library/assertions_configuration.yml @@ -0,0 +1,76 @@ +version: 1 +namespace: test-config-id-1 +assertions: + # Freshness Assertion + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.test_assertions_all_times,PROD) + type: freshness + lookback_interval: "1 hour" + last_modified_field: col_timestamp + schedule: + type: cron + cron: 0 * * * * + meta: + entity_qualified_name: TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES + entity_schema: + - col: col_date + native_type: DATE + # Volume Assertion + - type: volume + entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.test_assertions_all_times,PROD) + metric: row_count + condition: + type: less_than_or_equal_to + value: 1000 + schedule: + type: cron + cron: 0 * * * * + meta: + entity_qualified_name: TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES + entity_schema: + - col: col_date + native_type: DATE + # Field Metric Assertion + - type: field + entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.test_assertions_all_times,PROD) + field: col_date + metric: null_count + condition: + type: equal_to + value: 0 + schedule: + type: cron + cron: 0 * * * * + meta: + entity_qualified_name: TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES + entity_schema: + - col: col_date + native_type: DATE + # Field Value Assertion + - type: field + entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_event,PROD) + field: quantity + condition: + type: between + min: 0 + max: 10 + schedule: + type: on_table_change + meta: + entity_qualified_name: TEST_DB.PUBLIC.PURCHASE_EVENT + entity_schema: + - col: quantity + native_type: FLOAT + # Custom SQL Metric Assertion + - type: sql + entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_event,PROD) + statement: select mode(quantity) from test_db.public.purchase_event + condition: + type: equal_to + value: 5 + schedule: + type: on_table_change + meta: + entity_qualified_name: TEST_DB.PUBLIC.PURCHASE_EVENT + entity_schema: + - col: quantity + native_type: FLOAT diff --git a/metadata-ingestion/examples/library/create_form.py b/metadata-ingestion/examples/library/create_form.py new file mode 100644 index 0000000000000..3ba545ead3ed8 --- /dev/null +++ b/metadata-ingestion/examples/library/create_form.py @@ -0,0 +1,56 @@ +import logging + +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter + +# Imports for metadata model classes +from datahub.metadata.schema_classes import ( + FormActorAssignmentClass, + FormInfoClass, + FormPromptClass, + FormPromptTypeClass, + FormTypeClass, + StructuredPropertyParamsClass, +) +from datahub.metadata.urns import FormUrn + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +# define the prompts for our form +prompt_1 = FormPromptClass( + id="1", # ensure IDs are globally unique + title="First Prompt", + type=FormPromptTypeClass.STRUCTURED_PROPERTY, # structured property type prompt + structuredPropertyParams=StructuredPropertyParamsClass( + urn="urn:li:structuredProperty:property1" + ), # reference existing structured property + required=True, +) +prompt_2 = FormPromptClass( + id="2", # ensure IDs are globally unique + title="Second Prompt", + type=FormPromptTypeClass.FIELDS_STRUCTURED_PROPERTY, # structured property prompt on dataset schema fields + structuredPropertyParams=StructuredPropertyParamsClass( + urn="urn:li:structuredProperty:property1" + ), + required=False, # dataset schema fields prompts should not be required +) + +form_urn = FormUrn("metadata_initiative_1") +form_info_aspect = FormInfoClass( + name="Metadata Initiative 2024", + description="Please respond to this form for metadata compliance purposes", + type=FormTypeClass.VERIFICATION, + actors=FormActorAssignmentClass(owners=True), + prompts=[prompt_1, prompt_2], +) + +event: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityUrn=str(form_urn), + aspect=form_info_aspect, +) + +# Create rest emitter +rest_emitter = DatahubRestEmitter(gms_server="http://localhost:8080") +rest_emitter.emit(event) diff --git a/metadata-ingestion/examples/library/dataset_add_structured_properties.py b/metadata-ingestion/examples/library/dataset_add_structured_properties.py new file mode 100644 index 0000000000000..fc2c379340592 --- /dev/null +++ b/metadata-ingestion/examples/library/dataset_add_structured_properties.py @@ -0,0 +1,24 @@ +import logging + +from datahub.emitter.mce_builder import make_dataset_urn +from datahub.emitter.rest_emitter import DataHubRestEmitter +from datahub.specific.dataset import DatasetPatchBuilder + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +# Create rest emitter +rest_emitter = DataHubRestEmitter(gms_server="http://localhost:8080") + +dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD") + + +for patch_mcp in ( + DatasetPatchBuilder(dataset_urn) + .add_structured_property("io.acryl.dataManagement.replicationSLA", 12) + .build() +): + rest_emitter.emit(patch_mcp) + + +log.info(f"Added cluster_name, retention_time properties to dataset {dataset_urn}") diff --git a/metadata-ingestion/examples/library/dataset_read_operations.py b/metadata-ingestion/examples/library/dataset_read_operations.py new file mode 100644 index 0000000000000..78c9a92141cef --- /dev/null +++ b/metadata-ingestion/examples/library/dataset_read_operations.py @@ -0,0 +1,19 @@ +from datahub.api.graphql import Operation + +DATAHUB_HOST = "https//:org.acryl.io/gms" +DATAHUB_TOKEN = ", + # end_time_millis= +) diff --git a/metadata-ingestion/examples/library/dataset_remove_structured_properties.py b/metadata-ingestion/examples/library/dataset_remove_structured_properties.py new file mode 100644 index 0000000000000..d9f3c50464db2 --- /dev/null +++ b/metadata-ingestion/examples/library/dataset_remove_structured_properties.py @@ -0,0 +1,24 @@ +import logging + +from datahub.emitter.mce_builder import make_dataset_urn +from datahub.emitter.rest_emitter import DataHubRestEmitter +from datahub.specific.dataset import DatasetPatchBuilder + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +# Create rest emitter +rest_emitter = DataHubRestEmitter(gms_server="http://localhost:8080") + +dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD") + + +for patch_mcp in ( + DatasetPatchBuilder(dataset_urn) + .remove_structured_property("io.acryl.dataManagement.replicationSLA") + .build() +): + rest_emitter.emit(patch_mcp) + + +log.info(f"Added cluster_name, retention_time properties to dataset {dataset_urn}") diff --git a/metadata-ingestion/examples/library/dataset_report_operation.py b/metadata-ingestion/examples/library/dataset_report_operation.py new file mode 100644 index 0000000000000..15ebc43dba60a --- /dev/null +++ b/metadata-ingestion/examples/library/dataset_report_operation.py @@ -0,0 +1,19 @@ +from datahub.api.graphql import Operation + +DATAHUB_HOST = "https//:org.acryl.io/gms" +DATAHUB_TOKEN = " Union[DataHubRestEmitter, DatahubKafkaEmitter]: + USE_REST_EMITTER = True + if USE_REST_EMITTER: + gms_endpoint = "http://localhost:8080" + return DataHubRestEmitter(gms_server=gms_endpoint) + else: + kafka_server = "localhost:9092" + schema_registry_url = "http://localhost:8081" + return DatahubKafkaEmitter( + config=KafkaEmitterConfig( + connection=KafkaProducerConnectionConfig( + bootstrap=kafka_server, schema_registry_url=schema_registry_url + ) + ) + ) + + +# input your unique form ID +form_urn = FormUrn("metadata_initiative_1") + +# example prompts to add, must reference an existing structured property +new_prompt = FormPromptClass( + id="abcd", + title="title", + type=FormPromptTypeClass.STRUCTURED_PROPERTY, + structuredPropertyParams=StructuredPropertyParamsClass( + "urn:li:structuredProperty:io.acryl.test" + ), + required=True, +) +new_prompt2 = FormPromptClass( + id="1234", + title="title", + type=FormPromptTypeClass.FIELDS_STRUCTURED_PROPERTY, + structuredPropertyParams=StructuredPropertyParamsClass( + "urn:li:structuredProperty:io.acryl.test" + ), + required=True, +) + +with get_emitter() as emitter: + for patch_mcp in ( + FormPatchBuilder(str(form_urn)) + .add_owner( + OwnerClass( + owner="urn:li:corpuser:jdoe", type=OwnershipTypeClass.TECHNICAL_OWNER + ) + ) + .set_name("New Name") + .set_description("New description here") + .set_type(FormTypeClass.VERIFICATION) + .set_ownership_form(True) + .add_prompts([new_prompt, new_prompt2]) + .build() + ): + emitter.emit(patch_mcp) diff --git a/metadata-ingestion/examples/library/upsert_custom_assertion.py b/metadata-ingestion/examples/library/upsert_custom_assertion.py new file mode 100644 index 0000000000000..1878a894c0455 --- /dev/null +++ b/metadata-ingestion/examples/library/upsert_custom_assertion.py @@ -0,0 +1,28 @@ +import logging + +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +log = logging.getLogger(__name__) + +graph = DataHubGraph( + config=DatahubClientConfig( + server="http://localhost:8080", + ) +) + +new_assertion_urn = "urn:li:assertion:my-unique-assertion-id" + +# Upsert the assertion +res = graph.upsert_custom_assertion( + urn=new_assertion_urn, # If the assertion already exists, provide the URN + entity_urn="", + type="My Custom Category", # This categorizes your assertion in DataHub + description="The description of my external assertion for my dataset", + platform_urn="urn:li:dataPlatform:great-expectations", # OR you can provide 'platformName="My Custom Platform"' + field_path="field_foo", # Optional: if you want to associate it with a specific field + external_url="https://my-monitoring-tool.com/result-for-this-assertion", # Optional: link to monitoring tool + logic="SELECT * FROM X WHERE Y", # Optional: custom SQL for the assertion, rendered in the UI +) + +if res is not None: + log.info(f"Upserted assertion with urn: {new_assertion_urn}") diff --git a/metadata-ingestion/examples/perf/lineage_perf_example.py b/metadata-ingestion/examples/perf/lineage_perf_example.py new file mode 100644 index 0000000000000..3ee78bacb268a --- /dev/null +++ b/metadata-ingestion/examples/perf/lineage_perf_example.py @@ -0,0 +1,402 @@ +from typing import Iterable + +from datahub.emitter.mce_builder import make_data_job_urn, make_dataset_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.graph.client import get_default_graph +from datahub.metadata.schema_classes import ( + DataJobInputOutputClass, + DatasetLineageTypeClass, + DatasetPropertiesClass, + List, + StatusClass, + UpstreamClass, + UpstreamLineageClass, +) +from datahub.utilities.urns.dataset_urn import DatasetUrn + + +def lineage_mcp_generator( + urn: str, upstreams: List[str] +) -> Iterable[MetadataChangeProposalWrapper]: + yield MetadataChangeProposalWrapper( + entityUrn=urn, + aspect=UpstreamLineageClass( + upstreams=[ + UpstreamClass( + dataset=upstream, + type=DatasetLineageTypeClass.TRANSFORMED, + ) + for upstream in upstreams + ] + ), + ) + for upstream in upstreams: + yield MetadataChangeProposalWrapper( + entityUrn=upstream, aspect=StatusClass(removed=False) + ) + for urn_itr in [urn, *upstreams]: + yield MetadataChangeProposalWrapper( + entityUrn=urn_itr, + aspect=DatasetPropertiesClass(name=DatasetUrn.from_string(urn_itr).name), + ) + + +def datajob_lineage_mcp_generator( + urn: str, upstreams: List[str], downstreams: List[str] +) -> Iterable[MetadataChangeProposalWrapper]: + yield MetadataChangeProposalWrapper( + entityUrn=urn, + aspect=DataJobInputOutputClass( + inputDatasets=upstreams, + outputDatasets=downstreams, + ), + ) + for upstream in upstreams: + yield MetadataChangeProposalWrapper( + entityUrn=upstream, aspect=StatusClass(removed=False) + ) + for downstream in downstreams: + yield MetadataChangeProposalWrapper( + entityUrn=downstream, aspect=StatusClass(removed=False) + ) + + +def scenario_truncate_basic(): + """searchAcrossLineage(root, depth=n, breadth=3, skip=None) + All 21 urns. + """ + + path = "truncate.basic" + root_urn = make_dataset_urn("snowflake", f"{path}.root") + + yield from lineage_mcp_generator( + root_urn, + [make_dataset_urn("snowflake", f"{path}.u_{i}") for i in range(10)], + ) + + for i in range(10): + yield from lineage_mcp_generator( + make_dataset_urn("snowflake", f"{path}.d_{i}"), [root_urn] + ) + + +def scenario_truncate_intermediate(): + """searchAcrossLineage(root, depth=3, skip=None) + 1 root urn, all 3 direct upstreams and downstreams, and then 4 urns for each 'expanded' urn. + Total 1 + 3 + 4*3 = 16 urns. + """ + + path = "truncate.intermediate" + root_urn = make_dataset_urn("snowflake", f"{path}.root") + + yield from lineage_mcp_generator( + root_urn, [make_dataset_urn("snowflake", f"{path}.u_{i}") for i in range(10)] + ) + + for i in range(3): + yield from lineage_mcp_generator( + make_dataset_urn("snowflake", f"{path}.u_{i}"), + [make_dataset_urn("snowflake", f"{path}.u_{i}_u_{j}") for j in range(3)], + ) + + for i in range(3): + yield from lineage_mcp_generator( + make_dataset_urn("snowflake", f"{path}.d_{i}"), [root_urn] + ) + for j in range(3): + yield from lineage_mcp_generator( + make_dataset_urn("snowflake", f"{path}.d_{i}d_{j}"), + [make_dataset_urn("snowflake", f"{path}.d_{i}")], + ) + + +def scenario_truncate_complex(): + """searchAcrossLineage(root, depth=n, breadth=3, skip=None) + 1 root urn, + direct (lvl a) upstream, + its two (lvl b) upstreams, + each of their 3 (lvl c) upstreams, + each of their 4 (lvl d) upstreams, + then, for three of the lvl d nodes, 5 (lvl e) upstreams each, + then, for two of the lvl e nodes, 6 (lvl f) upstreams, and for the other lvl e node, 1 (lvl f) upstream. + Total 1 + 1 + 2 + (2 * 3) + (2 * 3 * 4) + (2 * 3 * 3 * 5) + (2 * 3 * 3 * 2 * 6) + (2 * 3 * 3 * 1 * 1) = 358 urns. + """ + + path = "truncate.complex" + root_urn = make_dataset_urn("snowflake", f"{path}.root") + lvl_a = make_dataset_urn("snowflake", f"{path}.u_0") + lvl_b = {i: make_dataset_urn("snowflake", f"{path}.u_0_u_{i}") for i in range(2)} + lvl_c = { + (a, b): make_dataset_urn("snowflake", f"{path}.u_0_u_{a}_u_{b}") + for a in range(2) + for b in range(3) + } + lvl_d = { + (a, b, c): make_dataset_urn("snowflake", f"{path}.u_0_u_{a}_u_{b}_u_{c}") + for a in range(2) + for b in range(3) + for c in range(4) + } + lvl_e = { + (a, b, c, d): make_dataset_urn( + "snowflake", f"{path}.u_0_u_{a}_u_{b}_u_{c}_u_{d}" + ) + for a in range(2) + for b in range(3) + for c in range(4) + for d in range(5) + } + lvl_f = { + (a, b, c, d, e): make_dataset_urn( + "snowflake", f"{path}.u_0_u_{a}_u_{b}_u_{c}_u_{d}_u_{e}" + ) + for a in range(2) + for b in range(3) + for c in range(4) + for d in range(5) + for e in range(6 if d % 2 == 0 else 1) + } + + yield from lineage_mcp_generator(root_urn, [lvl_a]) + yield from lineage_mcp_generator(lvl_a, list(lvl_b.values())) + for a, urn in lvl_b.items(): + yield from lineage_mcp_generator(urn, [lvl_c[(a, b)] for b in range(3)]) + for (a, b), urn in lvl_c.items(): + yield from lineage_mcp_generator(urn, [lvl_d[(a, b, c)] for c in range(4)]) + for (a, b, c), urn in lvl_d.items(): + yield from lineage_mcp_generator(urn, [lvl_e[(a, b, c, d)] for d in range(5)]) + for (a, b, c, d), urn in lvl_e.items(): + yield from lineage_mcp_generator( + urn, [lvl_f[(a, b, c, d, e)] for e in range(6 if d % 2 == 0 else 1)] + ) + + +def scenario_skip_basic(): + """searchAcrossLineage(root, depth=1, breadth=10, skip=[{type: "dataJob"}, {type: "dataset", platform: "urn:li:dataPlatform:dbt"}]) + 1 root urn, both airflow nodes, both dbt nodes, and all 6 snowflake neighbors. + Total 1 + 2 + 2 + 6 = 11 urns. + """ + path = "skip.basic" + root_urn = make_dataset_urn("snowflake", f"{path}.root") + upstream_dbt_urn = make_dataset_urn("dbt", f"{path}.u_0") + upstream_airflow_urn = make_data_job_urn("airflow", f"{path}.flow", f"{path}.u_0") + + yield from lineage_mcp_generator( + root_urn, + [ + make_dataset_urn("snowflake", f"{path}.u_direct"), + upstream_dbt_urn, + ], + ) + yield from lineage_mcp_generator( + upstream_dbt_urn, + [make_dataset_urn("snowflake", f"{path}.u_through_dbt")], + ) + yield from datajob_lineage_mcp_generator( + upstream_airflow_urn, + [make_dataset_urn("snowflake", f"{path}.u_through_airflow")], + [root_urn], + ) + + downstream_dbt_urn = make_dataset_urn("dbt", f"{path}.d_0") + downstream_airflow_urn = make_data_job_urn("airflow", f"{path}.flow", f"{path}.d_0") + yield from lineage_mcp_generator( + make_dataset_urn("snowflake", f"{path}.d_direct"), + [root_urn], + ) + yield from lineage_mcp_generator( + downstream_dbt_urn, + [root_urn], + ) + yield from lineage_mcp_generator( + make_dataset_urn("snowflake", f"{path}.d_through_dbt"), + [downstream_dbt_urn], + ) + yield from datajob_lineage_mcp_generator( + downstream_airflow_urn, + [root_urn], + [make_dataset_urn("snowflake", f"{path}.d_through_airflow")], + ) + + +def scenario_skip_intermediate(): + """searchAcrossLineage(root, depth=1, breadth=10, skip=[{type: "dataJob"}, {type: "dataset", platform: "urn:li:dataPlatform:dbt"}]) + 1 root urn and all nodes aside from those upstream of `skip.intermediate.u_indirect_1`. + Total 11 urns. + searchAcrossLineage(root, depth=2, breadth=10, skip=[{type: "dataJob"}, {type: "dataset", platform: "urn:li:dataPlatform:dbt"}]) + All 14 urns. + """ + path = "skip.intermediate" + root_urn = make_dataset_urn("snowflake", f"{path}.root") + upstream_dbt_urns = [make_dataset_urn("dbt", f"{path}.u_{i}") for i in range(6)] + upstream_airflow_urn = make_data_job_urn("airflow", f"{path}.flow", f"{path}.u_0") + + yield from lineage_mcp_generator( + root_urn, + [ + make_dataset_urn("snowflake", f"{path}.u_direct"), + upstream_dbt_urns[0], + ], + ) + yield from datajob_lineage_mcp_generator( + upstream_airflow_urn, [upstream_dbt_urns[1]], [upstream_dbt_urns[0]] + ) + yield from lineage_mcp_generator( + upstream_dbt_urns[1], + [ + upstream_dbt_urns[2], + ], + ) + yield from lineage_mcp_generator( + upstream_dbt_urns[2], + [ + upstream_dbt_urns[3], + upstream_dbt_urns[4], + ], + ) + yield from lineage_mcp_generator( + upstream_dbt_urns[3], + [make_dataset_urn("snowflake", f"{path}.u_indirect_0")], + ) + yield from lineage_mcp_generator( + upstream_dbt_urns[4], + [ + make_dataset_urn("snowflake", f"{path}.u_indirect_1"), + make_dataset_urn("snowflake", f"{path}.u_indirect_2"), + ], + ) + yield from lineage_mcp_generator( + make_dataset_urn("snowflake", f"{path}.u_indirect_1"), + [make_dataset_urn("snowflake", f"{path}.u_depth_2"), upstream_dbt_urns[5]], + ) + yield from lineage_mcp_generator( + upstream_dbt_urns[5], + [ + make_dataset_urn("snowflake", f"{path}.u_depth_2_indirect"), + ], + ) + + +def scenario_skip_complex(): + """searchAcrossLineage(root, depth=1, breadth=1, skip=[{type: "dataJob"}, {type: "dataset", platform: "urn:li:dataPlatform:dbt"}]) + The 11 urns from scenario_skip_intermediate, plus 2 snowflake urns and 1 dbt node from the single expanded upstream. + Total 14 urns. + """ + path = "skip.complex" + root_urn = make_dataset_urn("snowflake", f"{path}.root") + upstream_dbt_urns = [make_dataset_urn("dbt", f"{path}.u_{i}") for i in range(5)] + upstream_airflow_urn = make_data_job_urn("airflow", f"{path}.flow", f"{path}.u_0") + depth_one_snowflake_urns = { + "direct": make_dataset_urn("snowflake", f"{path}.u_direct"), + "indirect_0": make_dataset_urn("snowflake", f"{path}.u_indirect_0"), + "indirect_1": make_dataset_urn("snowflake", f"{path}.u_indirect_1"), + "indirect_2": make_dataset_urn("snowflake", f"{path}.u_indirect_2"), + } + + yield from lineage_mcp_generator( + root_urn, + [ + depth_one_snowflake_urns["direct"], + upstream_dbt_urns[0], + ], + ) + yield from datajob_lineage_mcp_generator( + upstream_airflow_urn, [upstream_dbt_urns[1]], [upstream_dbt_urns[0]] + ) + yield from lineage_mcp_generator( + upstream_dbt_urns[1], + [ + upstream_dbt_urns[2], + ], + ) + yield from lineage_mcp_generator( + upstream_dbt_urns[2], + [ + upstream_dbt_urns[3], + upstream_dbt_urns[4], + ], + ) + yield from lineage_mcp_generator( + upstream_dbt_urns[3], + [depth_one_snowflake_urns["indirect_0"]], + ) + yield from lineage_mcp_generator( + upstream_dbt_urns[4], + [ + depth_one_snowflake_urns["indirect_1"], + depth_one_snowflake_urns["indirect_2"], + ], + ) + + for name, urn in depth_one_snowflake_urns.items(): + dbt_urn = make_dataset_urn("dbt", f"{path}.u_{name}") + yield from lineage_mcp_generator( + urn, + [make_dataset_urn("snowflake", f"{path}.direct_u_{name}"), dbt_urn], + ) + yield from lineage_mcp_generator( + dbt_urn, + [make_dataset_urn("snowflake", f"{path}.indirect_u_{name}")], + ) + + +def scenario_perf(): + """searchAcrossLineage(root, depth=n, breadth=3, skip=None) + 1 root urn, + direct (lvl a) upstream, + its 100 (lvl b) upstreams, + each of their 30 (lvl c) upstreams, + each of their 40 (lvl d) upstreams, + then, 50 (lvl e) upstreams each, + then, half of lvl e nodes, 6 (lvl f) upstreams, and for the other lvl e node, 1 (lvl f) upstream. + Total 1 + 1 + 100 + (100 * 30) + (100 * 30 * 40) + (100 * 30 * 40 * 5) = 723,102 urns. + Disabled by default to avoid overloading + """ + + path = "lineage.perf" + root_urn = make_dataset_urn("snowflake", f"{path}.root") + lvl_a = make_dataset_urn("snowflake", f"{path}.u_0") + lvl_b = {i: make_dataset_urn("snowflake", f"{path}.u_0_u_{i}") for i in range(100)} + lvl_c = { + (a, b): make_dataset_urn("snowflake", f"{path}.u_0_u_{a}_u_{b}") + for a in range(100) + for b in range(30) + } + lvl_d = { + (a, b, c): make_dataset_urn("snowflake", f"{path}.u_0_u_{a}_u_{b}_u_{c}") + for a in range(100) + for b in range(30) + for c in range(40) + } + lvl_e = { + (a, b, c, d): make_dataset_urn( + "snowflake", f"{path}.u_0_u_{a}_u_{b}_u_{c}_u_{d}" + ) + for a in range(100) + for b in range(30) + for c in range(40) + for d in range(5) + } + + yield from lineage_mcp_generator(root_urn, [lvl_a]) + yield from lineage_mcp_generator(lvl_a, list(lvl_b.values())) + for a, urn in lvl_b.items(): + yield from lineage_mcp_generator(urn, [lvl_c[(a, b)] for b in range(30)]) + for (a, b), urn in lvl_c.items(): + yield from lineage_mcp_generator(urn, [lvl_d[(a, b, c)] for c in range(40)]) + for (a, b, c), urn in lvl_d.items(): + yield from lineage_mcp_generator(urn, [lvl_e[(a, b, c, d)] for d in range(5)]) + + +if __name__ == "__main__": + graph = get_default_graph() + for mcp in [ + *scenario_truncate_basic(), + *scenario_truncate_intermediate(), + *scenario_truncate_complex(), + *scenario_skip_basic(), + *scenario_skip_intermediate(), + *scenario_skip_complex(), + # *scenario_perf(), + ]: + graph.emit_mcp(mcp) diff --git a/metadata-ingestion/examples/recipes/bigquery_to_datahub.dhub.yaml b/metadata-ingestion/examples/recipes/bigquery_to_datahub.dhub.yaml index 84f098fa06c5c..86f4898d9d502 100644 --- a/metadata-ingestion/examples/recipes/bigquery_to_datahub.dhub.yaml +++ b/metadata-ingestion/examples/recipes/bigquery_to_datahub.dhub.yaml @@ -16,6 +16,7 @@ source: #include_tables: true #include_views: true #include_table_lineage: true + #extract_policy_tags_from_catalog: true #start_time: 2021-12-15T20:08:23.091Z #end_time: 2023-12-15T20:08:23.091Z #profiling: diff --git a/metadata-ingestion/examples/structured_properties/create_structured_property.py b/metadata-ingestion/examples/structured_properties/create_structured_property.py new file mode 100644 index 0000000000000..e66ac3aec4122 --- /dev/null +++ b/metadata-ingestion/examples/structured_properties/create_structured_property.py @@ -0,0 +1,98 @@ +import logging + +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter + +# Imports for metadata model classes +from datahub.metadata.schema_classes import ( + PropertyValueClass, + StructuredPropertyDefinitionClass, +) +from datahub.metadata.urns import StructuredPropertyUrn + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +# Create rest emitter +rest_emitter = DatahubRestEmitter(gms_server="http://localhost:8080") + +# first, let's make an open ended structured property that allows one text value +text_property_urn = StructuredPropertyUrn("openTextProperty") +text_property_definition = StructuredPropertyDefinitionClass( + qualifiedName="io.acryl.openTextProperty", + displayName="Open Text Property", + valueType="urn:li:dataType:datahub.string", + cardinality="SINGLE", + entityTypes=[ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.container", + ], + description="This structured property allows a signle open ended response as a value", + immutable=False, +) + +event_prop_1: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityUrn=str(text_property_urn), + aspect=text_property_definition, +) +rest_emitter.emit(event_prop_1) + +# next, let's make a property that allows for multiple datahub entity urns as values +# This example property could be used to reference other users or groups in datahub +urn_property_urn = StructuredPropertyUrn("dataSteward") +urn_property_definition = StructuredPropertyDefinitionClass( + qualifiedName="io.acryl.dataManagement.dataSteward", + displayName="Data Steward", + valueType="urn:li:dataType:datahub.urn", + cardinality="MULTIPLE", + entityTypes=["urn:li:entityType:datahub.dataset"], + description="The data stewards of this asset are in charge of ensuring data cleanliness and governance", + immutable=True, + typeQualifier={ + "allowedTypes": [ + "urn:li:entityType:datahub.corpuser", + "urn:li:entityType:datahub.corpGroup", + ] + }, # this line ensures only user or group urns can be assigned as values +) + +event_prop_2: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityUrn=str(urn_property_urn), + aspect=urn_property_definition, +) +rest_emitter.emit(event_prop_2) + +# finally, let's make a single select number property with a few allowed options +number_property_urn = StructuredPropertyUrn("replicationSLA") +number_property_definition = StructuredPropertyDefinitionClass( + qualifiedName="io.acryl.dataManagement.replicationSLA", + displayName="Retention Time", + valueType="urn:li:dataType:datahub.number", + cardinality="SINGLE", + entityTypes=[ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow", + ], + description="SLA for how long data can be delayed before replicating to the destination cluster", + immutable=False, + allowedValues=[ + PropertyValueClass( + value=30, + description="30 days, usually reserved for datasets that are ephemeral and contain pii", + ), + PropertyValueClass( + value=90, + description="Use this for datasets that drive monthly reporting but contain pii", + ), + PropertyValueClass( + value=365, + description="Use this for non-sensitive data that can be retained for longer", + ), + ], +) + +event_prop_3: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityUrn=str(number_property_urn), + aspect=number_property_definition, +) +rest_emitter.emit(event_prop_3) diff --git a/metadata-ingestion/examples/structured_properties/update_structured_property.py b/metadata-ingestion/examples/structured_properties/update_structured_property.py new file mode 100644 index 0000000000000..9b80ebc236d8b --- /dev/null +++ b/metadata-ingestion/examples/structured_properties/update_structured_property.py @@ -0,0 +1,43 @@ +import logging +from typing import Union + +from datahub.configuration.kafka import KafkaProducerConnectionConfig +from datahub.emitter.kafka_emitter import DatahubKafkaEmitter, KafkaEmitterConfig +from datahub.emitter.rest_emitter import DataHubRestEmitter +from datahub.metadata.urns import StructuredPropertyUrn +from datahub.specific.structured_property import StructuredPropertyPatchBuilder + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + + +# Get an emitter, either REST or Kafka, this example shows you both +def get_emitter() -> Union[DataHubRestEmitter, DatahubKafkaEmitter]: + USE_REST_EMITTER = True + if USE_REST_EMITTER: + gms_endpoint = "http://localhost:8080" + return DataHubRestEmitter(gms_server=gms_endpoint) + else: + kafka_server = "localhost:9092" + schema_registry_url = "http://localhost:8081" + return DatahubKafkaEmitter( + config=KafkaEmitterConfig( + connection=KafkaProducerConnectionConfig( + bootstrap=kafka_server, schema_registry_url=schema_registry_url + ) + ) + ) + + +# input your unique structured property ID +property_urn = StructuredPropertyUrn("dataSteward") + +with get_emitter() as emitter: + for patch_mcp in ( + StructuredPropertyPatchBuilder(str(property_urn)) + .set_display_name("test display name") + .set_cardinality("MULTIPLE") + .add_entity_type("urn:li:entityType:datahub.dataJob") + .build() + ): + emitter.emit(patch_mcp) diff --git a/metadata-ingestion/scripts/avro_codegen.py b/metadata-ingestion/scripts/avro_codegen.py index cf87b303643d1..5be8b6ed4cc21 100644 --- a/metadata-ingestion/scripts/avro_codegen.py +++ b/metadata-ingestion/scripts/avro_codegen.py @@ -580,7 +580,7 @@ def generate_urn_class(entity_type: str, key_aspect: dict) -> str: init_coercion = "" init_validation = "" for field in fields: - init_validation += f'if not {field_name(field)}:\n raise InvalidUrnError("{field_name(field)} cannot be empty")\n' + init_validation += f'if not {field_name(field)}:\n raise InvalidUrnError("{class_name} {field_name(field)} cannot be empty")\n' # Generalized mechanism for validating embedded urns. field_urn_type_class = None @@ -600,7 +600,8 @@ def generate_urn_class(entity_type: str, key_aspect: dict) -> str: ) else: init_validation += ( - f"assert not UrnEncoder.contains_reserved_char({field_name(field)})\n" + f"if UrnEncoder.contains_reserved_char({field_name(field)}):\n" + f" raise InvalidUrnError(f'{class_name} {field_name(field)} contains reserved characters')\n" ) if field_name(field) == "env": diff --git a/metadata-ingestion/scripts/docgen.py b/metadata-ingestion/scripts/docgen.py index d240f8e16c770..797a2f698c2f4 100644 --- a/metadata-ingestion/scripts/docgen.py +++ b/metadata-ingestion/scripts/docgen.py @@ -583,6 +583,12 @@ def generate( if source and source != plugin_name: continue + if plugin_name in { + "snowflake-summary", + }: + logger.info(f"Skipping {plugin_name} as it is on the deny list") + continue + metrics["plugins"]["discovered"] = metrics["plugins"]["discovered"] + 1 # type: ignore # We want to attempt to load all plugins before printing a summary. source_type = None @@ -885,11 +891,14 @@ def generate( os.makedirs(source_dir, exist_ok=True) doc_file = f"{source_dir}/lineage-feature-guide.md" with open(doc_file, "w+") as f: - f.write("import FeatureAvailability from '@site/src/components/FeatureAvailability';\n\n") + f.write( + "import FeatureAvailability from '@site/src/components/FeatureAvailability';\n\n" + ) f.write(f"# About DataHub Lineage\n\n") f.write("\n") - f.write(""" + f.write( + """ Data lineage is a **map that shows how data flows through your organization.** It details where your data originates, how it travels, and where it ultimately ends up. This can happen within a single system (like data moving between Snowflake tables) or across various platforms. @@ -979,24 +988,27 @@ def generate( ### Automatic Lineage Extraction Support -This is a summary of automatic lineage extraciton support in our data source. Please refer to the **Important Capabilities** table in the source documentation. Note that even if the source does not support automatic extraction, you can still add lineage manually using our API & SDKs.\n""") +This is a summary of automatic lineage extraciton support in our data source. Please refer to the **Important Capabilities** table in the source documentation. Note that even if the source does not support automatic extraction, you can still add lineage manually using our API & SDKs.\n""" + ) - f.write("\n| Source | Table-Level Lineage | Column-Level Lineage | Related Configs |\n") + f.write( + "\n| Source | Table-Level Lineage | Column-Level Lineage | Related Configs |\n" + ) f.write("| ---------- | ------ | ----- |----- |\n") for platform_id, platform_docs in sorted( - source_documentation.items(), - key=lambda x: (x[1]["name"].casefold(), x[1]["name"]) - if "name" in x[1] - else (x[0].casefold(), x[0]), + source_documentation.items(), + key=lambda x: (x[1]["name"].casefold(), x[1]["name"]) + if "name" in x[1] + else (x[0].casefold(), x[0]), ): for plugin, plugin_docs in sorted( - platform_docs["plugins"].items(), - key=lambda x: str(x[1].get("doc_order")) - if x[1].get("doc_order") - else x[0], + platform_docs["plugins"].items(), + key=lambda x: str(x[1].get("doc_order")) + if x[1].get("doc_order") + else x[0], ): - platform_name = platform_docs['name'] + platform_name = platform_docs["name"] if len(platform_docs["plugins"].keys()) > 1: # We only need to show this if there are multiple modules. platform_name = f"{platform_name} `{plugin}`" @@ -1004,33 +1016,60 @@ def generate( # Initialize variables table_level_supported = "❌" column_level_supported = "❌" - config_names = '' + config_names = "" if "capabilities" in plugin_docs: plugin_capabilities = plugin_docs["capabilities"] for cap_setting in plugin_capabilities: capability_text = get_capability_text(cap_setting.capability) - capability_supported = get_capability_supported_badge(cap_setting.supported) + capability_supported = get_capability_supported_badge( + cap_setting.supported + ) - if capability_text == "Table-Level Lineage" and capability_supported == "✅": + if ( + capability_text == "Table-Level Lineage" + and capability_supported == "✅" + ): table_level_supported = "✅" - if capability_text == "Column-level Lineage" and capability_supported == "✅": + if ( + capability_text == "Column-level Lineage" + and capability_supported == "✅" + ): column_level_supported = "✅" if not (table_level_supported == "❌" and column_level_supported == "❌"): if "config_schema" in plugin_docs: - config_properties = json.loads(plugin_docs['config_schema']).get('properties', {}) - config_names = '
'.join( - [f'- {property_name}' for property_name in config_properties if 'lineage' in property_name]) - lineage_not_applicable_sources = ['azure-ad', 'csv', 'demo-data', 'dynamodb', 'iceberg', 'json-schema', 'ldap', 'openapi', 'pulsar', 'sqlalchemy' ] - if platform_id not in lineage_not_applicable_sources : + config_properties = json.loads( + plugin_docs["config_schema"] + ).get("properties", {}) + config_names = "
".join( + [ + f"- {property_name}" + for property_name in config_properties + if "lineage" in property_name + ] + ) + lineage_not_applicable_sources = [ + "azure-ad", + "csv", + "demo-data", + "dynamodb", + "iceberg", + "json-schema", + "ldap", + "openapi", + "pulsar", + "sqlalchemy", + ] + if platform_id not in lineage_not_applicable_sources: f.write( f"| [{platform_name}](../../generated/ingestion/sources/{platform_id}.md) | {table_level_supported} | {column_level_supported} | {config_names}|\n" ) - f.write(""" + f.write( + """ ### SQL Parser Lineage Extraction @@ -1054,10 +1093,12 @@ def generate( - [Data in Context: Lineage Explorer in DataHub](https://blog.datahubproject.io/data-in-context-lineage-explorer-in-datahub-a53a9a476dc4) - [Harnessing the Power of Data Lineage with DataHub](https://blog.datahubproject.io/harnessing-the-power-of-data-lineage-with-datahub-ad086358dec4) - [Data Lineage: What It Is And Why It Matters](https://blog.datahubproject.io/data-lineage-what-it-is-and-why-it-matters-1a8d9846f0bd) - """) + """ + ) print("Lineage Documentation Generation Complete") + if __name__ == "__main__": logger.setLevel("INFO") generate() diff --git a/metadata-ingestion/scripts/install_deps.sh b/metadata-ingestion/scripts/install_deps.sh index bae0278056ebb..80a07cb04cb44 100755 --- a/metadata-ingestion/scripts/install_deps.sh +++ b/metadata-ingestion/scripts/install_deps.sh @@ -18,7 +18,8 @@ else sqlite-devel \ xz-devel \ libxml2-devel \ - libxslt-devel + libxslt-devel \ + krb5-devel else $sudo_cmd apt-get update && $sudo_cmd apt-get install -y \ python3-ldap \ diff --git a/metadata-ingestion/scripts/modeldocgen.py b/metadata-ingestion/scripts/modeldocgen.py index ef6ce765c23ed..ea7813f0ca85b 100644 --- a/metadata-ingestion/scripts/modeldocgen.py +++ b/metadata-ingestion/scripts/modeldocgen.py @@ -504,6 +504,7 @@ class AspectPluginConfig: className: str enabled: bool supportedEntityAspectNames: List[EntityAspectName] + packageScan: Optional[List[str]] = None supportedOperations: Optional[List[str]] = None diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 2bfc94d13aa14..41c04ca4a433c 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -34,7 +34,7 @@ "importlib_metadata>=4.0.0; python_version < '3.10'", "docker", "expandvars>=0.6.5", - "avro-gen3==0.7.12", + "avro-gen3==0.7.13", # "avro-gen3 @ git+https://github.com/acryldata/avro_gen@master#egg=avro-gen3", "avro>=1.11.3,<1.12", "python-dateutil>=2.8.0", @@ -99,14 +99,18 @@ sqlglot_lib = { # Using an Acryl fork of sqlglot. # https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:main?expand=1 - "acryl-sqlglot[rs]==24.0.1.dev7", + "acryl-sqlglot[rs]==25.3.1.dev3", } classification_lib = { - "acryl-datahub-classify==0.0.10", + "acryl-datahub-classify==0.0.11", # This is a bit of a hack. Because we download the SpaCy model at runtime in the classify plugin, # we need pip to be available. "pip", + # We were seeing an error like this `numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject` + # with numpy 2.0. This likely indicates a mismatch between scikit-learn and numpy versions. + # https://stackoverflow.com/questions/40845304/runtimewarning-numpy-dtype-size-changed-may-indicate-binary-incompatibility + "numpy<2", } sql_common = ( @@ -159,15 +163,16 @@ # LookML files with spaces between an item and the following comma. # See https://github.com/joshtemple/lkml/issues/73. "lkml>=1.3.4", - "sql-metadata==2.2.2", - *sqllineage_lib, + *sqlglot_lib, "GitPython>2", + "python-liquid", } bigquery_common = { # Google cloud logging library "google-cloud-logging<=3.5.0", "google-cloud-bigquery", + "google-cloud-datacatalog>=1.5.0", "more-itertools>=8.12.0", "sqlalchemy-bigquery>=1.4.1", } @@ -197,6 +202,7 @@ "pandas", "cryptography", "msal", + "cachetools", } | classification_lib trino = { @@ -225,11 +231,7 @@ iceberg_common = { # Iceberg Python SDK - "pyiceberg~=0.4", - # We currently pin to pydantic v1, since we only test against pydantic v1 in CI. - # However, we should remove this once we fix compatibility with newer versions - # of pyiceberg, which depend on pydantic v2. - *pydantic_no_v2, + "pyiceberg>=0.4,<0.7", } mssql_common = { @@ -263,7 +265,9 @@ delta_lake = { *s3_base, - "deltalake>=0.6.3, != 0.6.4", + # Version 0.18.0 broken on ARM Macs: https://github.com/delta-io/delta-rs/issues/2577 + "deltalake>=0.6.3, != 0.6.4, < 0.18.0; platform_system == 'Darwin' and platform_machine == 'arm64'", + "deltalake>=0.6.3, != 0.6.4; platform_system != 'Darwin' or platform_machine != 'arm64'", } powerbi_report_server = {"requests", "requests_ntlm"} @@ -339,7 +343,9 @@ "feast": { "feast>=0.34.0,<1", "flask-openid>=1.3.0", + "dask[dataframe]<2024.7.0", }, + "grafana": {"requests"}, "glue": aws_common, # hdbcli is supported officially by SAP, sqlalchemy-hana is built on top but not officially supported "hana": sql_common @@ -404,6 +410,8 @@ "sagemaker": aws_common, "salesforce": {"simple-salesforce"}, "snowflake": snowflake_common | usage_common | sqlglot_lib, + "snowflake-summary": snowflake_common | usage_common | sqlglot_lib, + "snowflake-queries": snowflake_common | usage_common | sqlglot_lib, "sqlalchemy": sql_common, "sql-queries": usage_common | sqlglot_lib, "slack": slack, @@ -425,11 +433,11 @@ "nifi": {"requests", "packaging", "requests-gssapi"}, "powerbi": microsoft_common | {"lark[regex]==1.1.4", "sqlparse"} | sqlglot_lib, "powerbi-report-server": powerbi_report_server, - "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.8.1"}, + "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.8.2"}, "unity-catalog": databricks | sql_common | sqllineage_lib, # databricks is alias for unity-catalog and needs to be kept in sync "databricks": databricks | sql_common | sqllineage_lib, - "fivetran": snowflake_common | bigquery_common, + "fivetran": snowflake_common | bigquery_common | sqlglot_lib, "qlik-sense": sqlglot_lib | {"requests", "websocket-client"}, "sigma": sqlglot_lib | {"requests"}, } @@ -506,7 +514,7 @@ "flake8-tidy-imports>=4.3.0", "flake8-bugbear==23.3.12", "isort>=5.7.0", - "mypy==1.0.0", + "mypy==1.10.1", *test_api_requirements, pytest_dep, "pytest-asyncio>=0.16.0", @@ -628,6 +636,7 @@ "dynamodb = datahub.ingestion.source.dynamodb.dynamodb:DynamoDBSource", "elasticsearch = datahub.ingestion.source.elastic_search:ElasticsearchSource", "feast = datahub.ingestion.source.feast:FeastRepositorySource", + "grafana = datahub.ingestion.source.grafana.grafana_source:GrafanaSource", "glue = datahub.ingestion.source.aws.glue:GlueSource", "sagemaker = datahub.ingestion.source.aws.sagemaker:SagemakerSource", "hana = datahub.ingestion.source.sql.hana:HanaSource", @@ -655,6 +664,8 @@ "redshift = datahub.ingestion.source.redshift.redshift:RedshiftSource", "slack = datahub.ingestion.source.slack.slack:SlackSource", "snowflake = datahub.ingestion.source.snowflake.snowflake_v2:SnowflakeV2Source", + "snowflake-summary = datahub.ingestion.source.snowflake.snowflake_summary:SnowflakeSummarySource", + "snowflake-queries = datahub.ingestion.source.snowflake.snowflake_queries:SnowflakeQueriesSource", "superset = datahub.ingestion.source.superset:SupersetSource", "tableau = datahub.ingestion.source.tableau:TableauSource", "openapi = datahub.ingestion.source.openapi:OpenApiSource", @@ -709,6 +720,7 @@ "replace_external_url = datahub.ingestion.transformer.replace_external_url:ReplaceExternalUrl", "pattern_cleanup_dataset_usage_user = datahub.ingestion.transformer.pattern_cleanup_dataset_usage_user:PatternCleanupDatasetUsageUser", "domain_mapping_based_on_tags = datahub.ingestion.transformer.dataset_domain_based_on_tags:DatasetTagDomainMapper", + "tags_to_term = datahub.ingestion.transformer.tags_to_terms:TagsToTermMapper", ], "datahub.ingestion.sink.plugins": [ "file = datahub.ingestion.sink.file:FileSink", @@ -727,6 +739,11 @@ "file = datahub.ingestion.reporting.file_reporter:FileReporter", ], "datahub.custom_packages": [], + "datahub.fs.plugins": [ + "s3 = datahub.ingestion.fs.s3_fs:S3FileSystem", + "file = datahub.ingestion.fs.local_fs:LocalFileSystem", + "http = datahub.ingestion.fs.http_fs:HttpFileSystem", + ], } @@ -797,6 +814,7 @@ "datahub-kafka", "sync-file-emitter", "sql-parser", + "iceberg", } else set() ) diff --git a/metadata-ingestion/sink_docs/file.md b/metadata-ingestion/sink_docs/metadata-file.md similarity index 93% rename from metadata-ingestion/sink_docs/file.md rename to metadata-ingestion/sink_docs/metadata-file.md index 2991afacbd93d..7cac8d5542243 100644 --- a/metadata-ingestion/sink_docs/file.md +++ b/metadata-ingestion/sink_docs/metadata-file.md @@ -1,4 +1,4 @@ -# File +# Metadata File For context on getting started with ingestion, check out our [metadata ingestion guide](../README.md). @@ -10,7 +10,7 @@ Works with `acryl-datahub` out of the box. Outputs metadata to a file. This can be used to decouple metadata sourcing from the process of pushing it into DataHub, and is particularly useful for debugging purposes. -Note that the [file source](../../docs/generated/ingestion/sources/file.md) can read files generated by this sink. +Note that the [file source](../../docs/generated/ingestion/sources/metadata-file.md) can read files generated by this sink. ## Quickstart recipe @@ -35,4 +35,3 @@ Note that a `.` is used to denote nested fields in the YAML recipe. | Field | Required | Default | Description | | -------- | -------- | ------- | ------------------------- | | filename | ✅ | | Path to file to write to. | - diff --git a/metadata-ingestion/sink_overview.md b/metadata-ingestion/sink_overview.md index c71ba1f97932c..95f18a0a6cb94 100644 --- a/metadata-ingestion/sink_overview.md +++ b/metadata-ingestion/sink_overview.md @@ -25,7 +25,7 @@ When configuring ingestion for DataHub, you're likely to be sending the metadata For debugging purposes or troubleshooting, the following sinks can be useful: -- [File](sink_docs/file.md) +- [Metadata File](sink_docs/metadata-file.md) - [Console](sink_docs/console.md) ## Default Sink diff --git a/metadata-ingestion/src/datahub/api/entities/assertion/__init__.py b/metadata-ingestion/src/datahub/api/entities/assertion/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/metadata-ingestion/src/datahub/api/entities/assertion/assertion.py b/metadata-ingestion/src/datahub/api/entities/assertion/assertion.py new file mode 100644 index 0000000000000..e0975a1c0351c --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/assertion/assertion.py @@ -0,0 +1,57 @@ +from abc import abstractmethod +from typing import Optional + +from datahub.api.entities.assertion.assertion_trigger import AssertionTrigger +from datahub.configuration.pydantic_migration_helpers import v1_ConfigModel, v1_Field +from datahub.metadata.com.linkedin.pegasus2avro.assertion import AssertionInfo + + +class BaseAssertionProtocol(v1_ConfigModel): + @abstractmethod + def get_id(self) -> str: + pass + + @abstractmethod + def get_assertion_info_aspect( + self, + ) -> AssertionInfo: + pass + + @abstractmethod + def get_assertion_trigger( + self, + ) -> Optional[AssertionTrigger]: + pass + + +class BaseAssertion(v1_ConfigModel): + id_raw: Optional[str] = v1_Field( + default=None, + description="The raw id of the assertion." + "If provided, this is used when creating identifier for this assertion" + "along with assertion type and entity.", + ) + + id: Optional[str] = v1_Field( + default=None, + description="The id of the assertion." + "If provided, this is used as identifier for this assertion." + "If provided, no other assertion fields are considered to create identifier.", + ) + + description: Optional[str] = None + + # Can contain metadata extracted from datahub. e.g. + # - entity qualified name + # - entity schema + meta: Optional[dict] = None + + +class BaseEntityAssertion(BaseAssertion): + entity: str = v1_Field( + description="The entity urn that the assertion is associated with" + ) + + trigger: Optional[AssertionTrigger] = v1_Field( + description="The trigger schedule for assertion", alias="schedule" + ) diff --git a/metadata-ingestion/src/datahub/api/entities/assertion/assertion_config_spec.py b/metadata-ingestion/src/datahub/api/entities/assertion/assertion_config_spec.py new file mode 100644 index 0000000000000..08205cc621253 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/assertion/assertion_config_spec.py @@ -0,0 +1,41 @@ +from typing import List, Optional + +from ruamel.yaml import YAML +from typing_extensions import Literal + +from datahub.api.entities.assertion.datahub_assertion import DataHubAssertion +from datahub.configuration.pydantic_migration_helpers import v1_ConfigModel, v1_Field + + +class AssertionsConfigSpec(v1_ConfigModel): + """ + Declarative configuration specification for datahub assertions. + + This model is used as a simpler, Python-native representation to define assertions. + It can be easily parsed from a equivalent YAML file. + + Currently, this is converted into series of assertion MCPs that can be emitted to DataHub. + In future, this would invoke datahub GraphQL API to upsert assertions. + """ + + version: Literal[1] + + id: Optional[str] = v1_Field( + default=None, + alias="namespace", + description="Unique identifier of assertions configuration file", + ) + + assertions: List[DataHubAssertion] + + @classmethod + def from_yaml( + cls, + file: str, + ) -> "AssertionsConfigSpec": + with open(file) as fp: + yaml = YAML(typ="rt") # default, if not specfied, is 'rt' (round-trip) + orig_dictionary = yaml.load(fp) + parsed_spec = AssertionsConfigSpec.parse_obj(orig_dictionary) + # parsed_spec._original_yaml_dict = orig_dictionary + return parsed_spec diff --git a/metadata-ingestion/src/datahub/api/entities/assertion/assertion_operator.py b/metadata-ingestion/src/datahub/api/entities/assertion/assertion_operator.py new file mode 100644 index 0000000000000..8704ed13cb6c3 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/assertion/assertion_operator.py @@ -0,0 +1,304 @@ +import json +from typing import List, Optional, Union + +from typing_extensions import Literal, Protocol + +from datahub.configuration.pydantic_migration_helpers import v1_ConfigModel +from datahub.metadata.schema_classes import ( + AssertionStdOperatorClass, + AssertionStdParameterClass, + AssertionStdParametersClass, + AssertionStdParameterTypeClass, +) + + +class Operator(Protocol): + """Specification for an assertion operator. + + This class exists only for documentation (not used in typing checking). + """ + + operator: str + + def id(self) -> str: + ... + + def generate_parameters(self) -> AssertionStdParametersClass: + ... + + +def _generate_assertion_std_parameter( + value: Union[str, int, float, list] +) -> AssertionStdParameterClass: + if isinstance(value, str): + return AssertionStdParameterClass( + value=value, type=AssertionStdParameterTypeClass.STRING + ) + elif isinstance(value, (int, float)): + return AssertionStdParameterClass( + value=str(value), type=AssertionStdParameterTypeClass.NUMBER + ) + elif isinstance(value, list): + return AssertionStdParameterClass( + value=json.dumps(value), type=AssertionStdParameterTypeClass.LIST + ) + else: + raise ValueError( + f"Unsupported assertion parameter {value} of type {type(value)}" + ) + + +Param = Union[str, int, float, List[Union[str, float, int]]] + + +def _generate_assertion_std_parameters( + value: Optional[Param] = None, + min_value: Optional[Param] = None, + max_value: Optional[Param] = None, +) -> AssertionStdParametersClass: + return AssertionStdParametersClass( + value=_generate_assertion_std_parameter(value) if value else None, + minValue=_generate_assertion_std_parameter(min_value) if min_value else None, + maxValue=_generate_assertion_std_parameter(max_value) if max_value else None, + ) + + +class EqualToOperator(v1_ConfigModel): + type: Literal["equal_to"] + value: Union[str, int, float] + + operator: str = AssertionStdOperatorClass.EQUAL_TO + + def id(self) -> str: + return f"{self.type}-{self.value}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters(value=self.value) + + +class NotEqualToOperator(v1_ConfigModel): + type: Literal["not_equal_to"] + value: Union[str, int, float] + + operator: str = AssertionStdOperatorClass.NOT_EQUAL_TO + + def id(self) -> str: + return f"{self.type}-{self.value}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters(value=self.value) + + +class BetweenOperator(v1_ConfigModel): + type: Literal["between"] + min: Union[int, float] + max: Union[int, float] + + operator: str = AssertionStdOperatorClass.BETWEEN + + def id(self) -> str: + return f"{self.type}-{self.min}-{self.max}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters( + min_value=self.min, max_value=self.max + ) + + +class LessThanOperator(v1_ConfigModel): + type: Literal["less_than"] + value: Union[int, float] + + operator: str = AssertionStdOperatorClass.LESS_THAN + + def id(self) -> str: + return f"{self.type}-{self.value}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters(value=self.value) + + +class GreaterThanOperator(v1_ConfigModel): + type: Literal["greater_than"] + value: Union[int, float] + + operator: str = AssertionStdOperatorClass.GREATER_THAN + + def id(self) -> str: + return f"{self.type}-{self.value}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters(value=self.value) + + +class LessThanOrEqualToOperator(v1_ConfigModel): + type: Literal["less_than_or_equal_to"] + value: Union[int, float] + + operator: str = AssertionStdOperatorClass.LESS_THAN_OR_EQUAL_TO + + def id(self) -> str: + return f"{self.type}-{self.value}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters(value=self.value) + + +class GreaterThanOrEqualToOperator(v1_ConfigModel): + type: Literal["greater_than_or_equal_to"] + value: Union[int, float] + + operator: str = AssertionStdOperatorClass.GREATER_THAN_OR_EQUAL_TO + + def id(self) -> str: + return f"{self.type}-{self.value}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters(value=self.value) + + +class InOperator(v1_ConfigModel): + type: Literal["in"] + value: List[Union[str, float, int]] + + operator: str = AssertionStdOperatorClass.IN + + def id(self) -> str: + return f"{self.type}-{self.value}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters(value=self.value) + + +class NotInOperator(v1_ConfigModel): + type: Literal["not_in"] + value: List[Union[str, float, int]] + + operator: str = AssertionStdOperatorClass.NOT_IN + + def id(self) -> str: + return f"{self.type}-{self.value}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters(value=self.value) + + +class IsNullOperator(v1_ConfigModel): + type: Literal["is_null"] + + operator: str = AssertionStdOperatorClass.NULL + + def id(self) -> str: + return f"{self.type}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters() + + +class NotNullOperator(v1_ConfigModel): + type: Literal["is_not_null"] + + operator: str = AssertionStdOperatorClass.NOT_NULL + + def id(self) -> str: + return f"{self.type}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters() + + +class IsTrueOperator(v1_ConfigModel): + type: Literal["is_true"] + + operator: str = AssertionStdOperatorClass.IS_TRUE + + def id(self) -> str: + return f"{self.type}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters() + + +class IsFalseOperator(v1_ConfigModel): + type: Literal["is_false"] + + operator: str = AssertionStdOperatorClass.IS_FALSE + + def id(self) -> str: + return f"{self.type}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters() + + +class ContainsOperator(v1_ConfigModel): + type: Literal["contains"] + value: str + + operator: str = AssertionStdOperatorClass.CONTAIN + + def id(self) -> str: + return f"{self.type}-{self.value}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters(value=self.value) + + +class EndsWithOperator(v1_ConfigModel): + type: Literal["ends_with"] + value: str + + operator: str = AssertionStdOperatorClass.END_WITH + + def id(self) -> str: + return f"{self.type}-{self.value}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters(value=self.value) + + +class StartsWithOperator(v1_ConfigModel): + type: Literal["starts_with"] + value: str + + operator: str = AssertionStdOperatorClass.START_WITH + + def id(self) -> str: + return f"{self.type}-{self.value}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters(value=self.value) + + +class MatchesRegexOperator(v1_ConfigModel): + type: Literal["matches_regex"] + value: str + + operator: str = AssertionStdOperatorClass.REGEX_MATCH + + def id(self) -> str: + return f"{self.type}-{self.value}" + + def generate_parameters(self) -> AssertionStdParametersClass: + return _generate_assertion_std_parameters(value=self.value) + + +Operators = Union[ + InOperator, + NotInOperator, + EqualToOperator, + NotEqualToOperator, + BetweenOperator, + LessThanOperator, + LessThanOrEqualToOperator, + GreaterThanOperator, + GreaterThanOrEqualToOperator, + IsNullOperator, + NotNullOperator, + IsTrueOperator, + IsFalseOperator, + ContainsOperator, + EndsWithOperator, + StartsWithOperator, + MatchesRegexOperator, +] diff --git a/metadata-ingestion/src/datahub/api/entities/assertion/assertion_trigger.py b/metadata-ingestion/src/datahub/api/entities/assertion/assertion_trigger.py new file mode 100644 index 0000000000000..d780916484744 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/assertion/assertion_trigger.py @@ -0,0 +1,52 @@ +from datetime import timedelta +from typing import Union + +import humanfriendly +from typing_extensions import Literal + +from datahub.configuration.pydantic_migration_helpers import ( + v1_ConfigModel, + v1_Field, + v1_validator, +) + + +class CronTrigger(v1_ConfigModel): + type: Literal["cron"] + cron: str = v1_Field( + description="The cron expression to use. See https://crontab.guru/ for help." + ) + timezone: str = v1_Field( + "UTC", + description="The timezone to use for the cron schedule. Defaults to UTC.", + ) + + +class IntervalTrigger(v1_ConfigModel): + type: Literal["interval"] + interval: timedelta + + @v1_validator("interval", pre=True) + def lookback_interval_to_timedelta(cls, v): + if isinstance(v, str): + seconds = humanfriendly.parse_timespan(v) + return timedelta(seconds=seconds) + raise ValueError("Invalid value.") + + +class EntityChangeTrigger(v1_ConfigModel): + type: Literal["on_table_change"] + + +class ManualTrigger(v1_ConfigModel): + type: Literal["manual"] + + +class AssertionTrigger(v1_ConfigModel): + __root__: Union[ + CronTrigger, IntervalTrigger, EntityChangeTrigger, ManualTrigger + ] = v1_Field(discriminator="type") + + @property + def trigger(self): + return self.__root__ diff --git a/metadata-ingestion/src/datahub/api/entities/assertion/compiler_interface.py b/metadata-ingestion/src/datahub/api/entities/assertion/compiler_interface.py new file mode 100644 index 0000000000000..27b43a58530b1 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/assertion/compiler_interface.py @@ -0,0 +1,81 @@ +from abc import abstractmethod +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Dict, List, Literal + +from datahub.api.entities.assertion.assertion_config_spec import AssertionsConfigSpec +from datahub.ingestion.api.report import Report +from datahub.utilities.lossy_collections import LossyDict, LossyList + + +class StrEnum(str, Enum): + pass + + +class CompileResultArtifactType(StrEnum): + SQL_QUERIES = "SQL_QUERIES" + COMPILE_REPORT = "COMPILE_REPORT" + + +@dataclass +class CompileResultArtifact(Report): + name: str + type: CompileResultArtifactType + path: Path + description: str + + +@dataclass +class AssertionCompilationReport(Report): + """Additional details to debug compilation""" + + num_processed: int = 0 + num_compile_succeeded: int = 0 + num_compile_failed: int = 0 # Likely due to assertion not supported in platform + + warnings: LossyDict[str, LossyList[str]] = field(default_factory=LossyDict) + failures: LossyDict[str, LossyList[str]] = field(default_factory=LossyDict) + + artifacts: List[Path] = field(default_factory=list) + + def report_warning(self, key: str, reason: str) -> None: + warnings = self.warnings.get(key, LossyList()) + warnings.append(reason) + self.warnings[key] = warnings + + def report_failure(self, key: str, reason: str) -> None: + failures = self.failures.get(key, LossyList()) + failures.append(reason) + self.failures[key] = failures + + +@dataclass +class AssertionCompilationResult: + """Results of compilation step , along with detailed report object""" + + platform: str + status: Literal["success", "failure"] + + report: AssertionCompilationReport = field( + default_factory=AssertionCompilationReport + ) + + artifacts: List[CompileResultArtifact] = field(default_factory=list) + + def add_artifact(self, artifact: CompileResultArtifact) -> None: + self.artifacts.append(artifact) + self.report.artifacts.append(artifact.path) + + +class AssertionCompiler: + @classmethod + @abstractmethod + def create(cls, output_dir: str, extras: Dict[str, str]) -> "AssertionCompiler": + pass + + @abstractmethod + def compile( + self, assertion_config_spec: AssertionsConfigSpec + ) -> AssertionCompilationResult: + pass diff --git a/metadata-ingestion/src/datahub/api/entities/assertion/datahub_assertion.py b/metadata-ingestion/src/datahub/api/entities/assertion/datahub_assertion.py new file mode 100644 index 0000000000000..ed18b78418d76 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/assertion/datahub_assertion.py @@ -0,0 +1,35 @@ +from typing import Optional, Union + +from datahub.api.entities.assertion.assertion import BaseAssertionProtocol +from datahub.api.entities.assertion.assertion_trigger import AssertionTrigger +from datahub.api.entities.assertion.field_assertion import FieldAssertion +from datahub.api.entities.assertion.freshness_assertion import FreshnessAssertion +from datahub.api.entities.assertion.sql_assertion import SQLAssertion +from datahub.api.entities.assertion.volume_assertion import VolumeAssertion +from datahub.configuration.pydantic_migration_helpers import v1_Field +from datahub.metadata.com.linkedin.pegasus2avro.assertion import AssertionInfo + + +class DataHubAssertion(BaseAssertionProtocol): + __root__: Union[ + FreshnessAssertion, + VolumeAssertion, + SQLAssertion, + FieldAssertion, + # TODO: Add SchemaAssertion + ] = v1_Field(discriminator="type") + + @property + def assertion(self): + return self.__root__.assertion + + def get_assertion_info_aspect( + self, + ) -> AssertionInfo: + return self.__root__.get_assertion_info_aspect() + + def get_id(self) -> str: + return self.__root__.get_id() + + def get_assertion_trigger(self) -> Optional[AssertionTrigger]: + return self.__root__.get_assertion_trigger() diff --git a/metadata-ingestion/src/datahub/api/entities/assertion/field_assertion.py b/metadata-ingestion/src/datahub/api/entities/assertion/field_assertion.py new file mode 100644 index 0000000000000..ae062c3a8e5cb --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/assertion/field_assertion.py @@ -0,0 +1,158 @@ +from enum import Enum +from typing import Optional, Union + +from typing_extensions import Literal + +from datahub.api.entities.assertion.assertion import ( + BaseAssertionProtocol, + BaseEntityAssertion, +) +from datahub.api.entities.assertion.assertion_operator import Operators +from datahub.api.entities.assertion.assertion_trigger import AssertionTrigger +from datahub.api.entities.assertion.field_metric import FieldMetric +from datahub.api.entities.assertion.filter import DatasetFilter +from datahub.configuration.pydantic_migration_helpers import v1_ConfigModel, v1_Field +from datahub.emitter.mce_builder import datahub_guid +from datahub.metadata.com.linkedin.pegasus2avro.assertion import ( + AssertionInfo, + AssertionType, + FieldAssertionInfo, + FieldAssertionType, +) +from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaFieldSpec +from datahub.metadata.schema_classes import ( + FieldMetricAssertionClass, + FieldTransformClass, + FieldTransformTypeClass, + FieldValuesAssertionClass, + FieldValuesFailThresholdClass, + FieldValuesFailThresholdTypeClass, +) + + +class FieldValuesFailThreshold(v1_ConfigModel): + type: Literal["count", "percentage"] = v1_Field(default="count") + value: int = v1_Field(default=0) + + def to_field_values_failure_threshold(self) -> FieldValuesFailThresholdClass: + return FieldValuesFailThresholdClass( + type=( + FieldValuesFailThresholdTypeClass.COUNT + if self.type == Literal["count"] + else FieldValuesFailThresholdTypeClass.PERCENTAGE + ), + value=self.value, + ) + + +class FieldTransform(Enum): + LENGTH = "length" + + +class FieldValuesAssertion(BaseEntityAssertion): + type: Literal["field"] + field: str + field_transform: Optional[FieldTransform] = v1_Field(default=None) + operator: Operators = v1_Field(discriminator="type", alias="condition") + filters: Optional[DatasetFilter] = v1_Field(default=None) + failure_threshold: FieldValuesFailThreshold = v1_Field( + default=FieldValuesFailThreshold() + ) + exclude_nulls: bool = v1_Field(default=True) + + def get_assertion_info( + self, + ) -> AssertionInfo: + return AssertionInfo( + description=self.description, + type=AssertionType.FIELD, + fieldAssertion=FieldAssertionInfo( + type=FieldAssertionType.FIELD_VALUES, + entity=self.entity, + fieldValuesAssertion=FieldValuesAssertionClass( + field=SchemaFieldSpec( + path=self.field, + type="", # Not required + nativeType="", # Not required + ), + operator=self.operator.operator, + parameters=self.operator.generate_parameters(), + failThreshold=self.failure_threshold.to_field_values_failure_threshold(), + excludeNulls=self.exclude_nulls, + transform=( + FieldTransformClass(type=FieldTransformTypeClass.LENGTH) + if self.field_transform == Literal["length"] + else None + ), + ), + ), + ) + + def get_id(self) -> str: + guid_dict = { + "entity": self.entity, + "type": self.type, + "field": self.field, + "operator": str(self.operator.operator), + "id_raw": self.id_raw, + } + return self.id or datahub_guid(guid_dict) + + +class FieldMetricAssertion(BaseEntityAssertion): + type: Literal["field"] + field: str + operator: Operators = v1_Field(discriminator="type", alias="condition") + metric: FieldMetric + filters: Optional[DatasetFilter] = v1_Field(default=None) + + def get_assertion_info( + self, + ) -> AssertionInfo: + return AssertionInfo( + description=self.description, + type=AssertionType.FIELD, + fieldAssertion=FieldAssertionInfo( + type=FieldAssertionType.FIELD_METRIC, + entity=self.entity, + fieldMetricAssertion=FieldMetricAssertionClass( + field=SchemaFieldSpec( + path=self.field, + type="", # Not required + nativeType="", # Not required + ), + metric=self.metric.name, + operator=self.operator.operator, + parameters=self.operator.generate_parameters(), + ), + ), + ) + + def get_id(self) -> str: + guid_dict = { + "entity": self.entity, + "type": self.type, + "field": self.field, + "metric": self.metric.value, + "id_raw": self.id_raw, + } + return self.id or datahub_guid(guid_dict) + + +class FieldAssertion(BaseAssertionProtocol): + __root__: Union[FieldMetricAssertion, FieldValuesAssertion] + + @property + def assertion(self): + return self.__root__ + + def get_id(self) -> str: + return self.__root__.get_id() + + def get_assertion_info_aspect( + self, + ) -> AssertionInfo: + return self.__root__.get_assertion_info() + + def get_assertion_trigger(self) -> Optional[AssertionTrigger]: + return self.__root__.trigger diff --git a/metadata-ingestion/src/datahub/api/entities/assertion/field_metric.py b/metadata-ingestion/src/datahub/api/entities/assertion/field_metric.py new file mode 100644 index 0000000000000..7a236da2d562d --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/assertion/field_metric.py @@ -0,0 +1,21 @@ +from enum import Enum + + +class FieldMetric(Enum): + UNIQUE_COUNT = "unique_count" + UNIQUE_PERCENTAGE = "unique_percentage" + NULL_COUNT = "null_count" + NULL_PERCENTAGE = "null_percentage" + MIN = "min" + MAX = "max" + MEAN = "mean" + MEDIAN = "median" + STDDEV = "stddev" + NEGATIVE_COUNT = "negative_count" + NEGATIVE_PERCENTAGE = "negative_percentage" + ZERO_COUNT = "zero_count" + ZERO_PERCENTAGE = "zero_percentage" + MIN_LENGTH = "min_length" + MAX_LENGTH = "max_length" + EMPTY_COUNT = "empty_count" + EMPTY_PERCENTAGE = "empty_percentage" diff --git a/metadata-ingestion/src/datahub/api/entities/assertion/filter.py b/metadata-ingestion/src/datahub/api/entities/assertion/filter.py new file mode 100644 index 0000000000000..05d75b674d6af --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/assertion/filter.py @@ -0,0 +1,13 @@ +from typing_extensions import Literal + +from datahub.configuration.pydantic_migration_helpers import v1_ConfigModel + + +class SqlFilter(v1_ConfigModel): + type: Literal["sql"] + sql: str + + +DatasetFilter = SqlFilter +# class DatasetFilter(v1_ConfigModel): +# __root__: Union[SqlFilter] = v1_Field(discriminator="type") diff --git a/metadata-ingestion/src/datahub/api/entities/assertion/freshness_assertion.py b/metadata-ingestion/src/datahub/api/entities/assertion/freshness_assertion.py new file mode 100644 index 0000000000000..f9e1df7d68f27 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/assertion/freshness_assertion.py @@ -0,0 +1,124 @@ +from datetime import timedelta +from enum import Enum +from typing import Optional, Union + +import humanfriendly +from typing_extensions import Literal + +from datahub.api.entities.assertion.assertion import ( + BaseAssertionProtocol, + BaseEntityAssertion, +) +from datahub.api.entities.assertion.assertion_trigger import AssertionTrigger +from datahub.api.entities.assertion.filter import DatasetFilter +from datahub.configuration.pydantic_migration_helpers import v1_Field, v1_validator +from datahub.emitter.mce_builder import datahub_guid +from datahub.metadata.com.linkedin.pegasus2avro.assertion import ( + AssertionInfo, + AssertionType, + FixedIntervalSchedule, + FreshnessAssertionInfo, + FreshnessAssertionSchedule, + FreshnessAssertionScheduleType, + FreshnessAssertionType, + FreshnessCronSchedule, +) +from datahub.metadata.com.linkedin.pegasus2avro.timeseries import CalendarInterval + + +class FreshnessSourceType(Enum): + LAST_MODIFIED_COLUMN = "last_modified_column" + + +class CronFreshnessAssertion(BaseEntityAssertion): + type: Literal["freshness"] + freshness_type: Literal["cron"] + cron: str = v1_Field( + description="The cron expression to use. See https://crontab.guru/ for help." + ) + timezone: str = v1_Field( + "UTC", + description="The timezone to use for the cron schedule. Defaults to UTC.", + ) + source_type: FreshnessSourceType = v1_Field( + default=FreshnessSourceType.LAST_MODIFIED_COLUMN + ) + last_modified_field: str + filters: Optional[DatasetFilter] = v1_Field(default=None) + + def get_assertion_info( + self, + ) -> AssertionInfo: + return AssertionInfo( + description=self.description, + type=AssertionType.FRESHNESS, + freshnessAssertion=FreshnessAssertionInfo( + type=FreshnessAssertionType.DATASET_CHANGE, + entity=self.entity, + schedule=FreshnessAssertionSchedule( + type=FreshnessAssertionScheduleType.CRON, + cron=FreshnessCronSchedule(cron=self.cron, timezone=self.timezone), + ), + ), + ) + + +class FixedIntervalFreshnessAssertion(BaseEntityAssertion): + type: Literal["freshness"] + freshness_type: Literal["interval"] = v1_Field(default="interval") + lookback_interval: timedelta + filters: Optional[DatasetFilter] = v1_Field(default=None) + source_type: FreshnessSourceType = v1_Field( + default=FreshnessSourceType.LAST_MODIFIED_COLUMN + ) + last_modified_field: str + + @v1_validator("lookback_interval", pre=True) + def lookback_interval_to_timedelta(cls, v): + if isinstance(v, str): + seconds = humanfriendly.parse_timespan(v) + return timedelta(seconds=seconds) + raise ValueError("Invalid value.") + + def get_assertion_info( + self, + ) -> AssertionInfo: + return AssertionInfo( + description=self.description, + type=AssertionType.FRESHNESS, + freshnessAssertion=FreshnessAssertionInfo( + type=FreshnessAssertionType.DATASET_CHANGE, + entity=self.entity, + schedule=FreshnessAssertionSchedule( + type=FreshnessAssertionScheduleType.FIXED_INTERVAL, + fixedInterval=FixedIntervalSchedule( + unit=CalendarInterval.SECOND, + multiple=self.lookback_interval.seconds, + ), + ), + ), + ) + + +class FreshnessAssertion(BaseAssertionProtocol): + __root__: Union[FixedIntervalFreshnessAssertion, CronFreshnessAssertion] + + @property + def assertion(self): + return self.__root__ + + def get_id(self) -> str: + guid_dict = { + "entity": self.__root__.entity, + "type": self.__root__.type, + "id_raw": self.__root__.id_raw, + } + return self.__root__.id or datahub_guid(guid_dict) + + def get_assertion_info_aspect( + self, + ) -> AssertionInfo: + return self.__root__.get_assertion_info() + + def get_assertion_trigger(self) -> Optional[AssertionTrigger]: + return self.__root__.trigger diff --git a/metadata-ingestion/src/datahub/api/entities/assertion/sql_assertion.py b/metadata-ingestion/src/datahub/api/entities/assertion/sql_assertion.py new file mode 100644 index 0000000000000..3d12cfde428f4 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/assertion/sql_assertion.py @@ -0,0 +1,91 @@ +from typing import Optional, Union + +from typing_extensions import Literal + +from datahub.api.entities.assertion.assertion import ( + BaseAssertionProtocol, + BaseEntityAssertion, +) +from datahub.api.entities.assertion.assertion_operator import Operators +from datahub.api.entities.assertion.assertion_trigger import AssertionTrigger +from datahub.configuration.pydantic_migration_helpers import v1_Field +from datahub.emitter.mce_builder import datahub_guid +from datahub.metadata.com.linkedin.pegasus2avro.assertion import ( + AssertionInfo, + AssertionType, + AssertionValueChangeType, + SqlAssertionInfo, + SqlAssertionType, +) + + +class SqlMetricAssertion(BaseEntityAssertion): + type: Literal["sql"] + statement: str + operator: Operators = v1_Field(discriminator="type", alias="condition") + + def get_assertion_info( + self, + ) -> AssertionInfo: + return AssertionInfo( + description=self.description, + type=AssertionType.SQL, + sqlAssertion=SqlAssertionInfo( + type=SqlAssertionType.METRIC, + entity=self.entity, + statement=self.statement, + operator=self.operator.operator, + parameters=self.operator.generate_parameters(), + ), + ) + + +class SqlMetricChangeAssertion(BaseEntityAssertion): + type: Literal["sql"] + statement: str + change_type: Literal["absolute", "percentage"] + operator: Operators = v1_Field(discriminator="type", alias="condition") + + def get_assertion_info( + self, + ) -> AssertionInfo: + return AssertionInfo( + description=self.description, + type=AssertionType.SQL, + sqlAssertion=SqlAssertionInfo( + type=SqlAssertionType.METRIC_CHANGE, + entity=self.entity, + statement=self.statement, + changeType=( + AssertionValueChangeType.ABSOLUTE + if self.change_type == Literal["absolute"] + else AssertionValueChangeType.PERCENTAGE + ), + operator=self.operator.operator, + parameters=self.operator.generate_parameters(), + ), + ) + + +class SQLAssertion(BaseAssertionProtocol): + __root__: Union[SqlMetricAssertion, SqlMetricChangeAssertion] = v1_Field() + + @property + def assertion(self): + return self.__root__ + + def get_id(self) -> str: + guid_dict = { + "entity": self.__root__.entity, + "type": self.__root__.type, + "id_raw": self.__root__.id_raw, + } + return self.__root__.id or datahub_guid(guid_dict) + + def get_assertion_info_aspect( + self, + ) -> AssertionInfo: + return self.__root__.get_assertion_info() + + def get_assertion_trigger(self) -> Optional[AssertionTrigger]: + return self.__root__.trigger diff --git a/metadata-ingestion/src/datahub/api/entities/assertion/volume_assertion.py b/metadata-ingestion/src/datahub/api/entities/assertion/volume_assertion.py new file mode 100644 index 0000000000000..da6a125874aa7 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/assertion/volume_assertion.py @@ -0,0 +1,98 @@ +from typing import Optional, Union + +from typing_extensions import Literal + +from datahub.api.entities.assertion.assertion import ( + BaseAssertionProtocol, + BaseEntityAssertion, +) +from datahub.api.entities.assertion.assertion_operator import Operators +from datahub.api.entities.assertion.assertion_trigger import AssertionTrigger +from datahub.api.entities.assertion.filter import DatasetFilter +from datahub.configuration.pydantic_migration_helpers import v1_Field +from datahub.emitter.mce_builder import datahub_guid +from datahub.metadata.com.linkedin.pegasus2avro.assertion import ( + AssertionInfo, + AssertionType, + AssertionValueChangeType, + RowCountChange, + RowCountTotal, + VolumeAssertionInfo, + VolumeAssertionType, +) + + +class RowCountTotalVolumeAssertion(BaseEntityAssertion): + type: Literal["volume"] + metric: Literal["row_count"] = v1_Field(default="row_count") + operator: Operators = v1_Field(discriminator="type", alias="condition") + filters: Optional[DatasetFilter] = v1_Field(default=None) + + def get_assertion_info( + self, + ) -> AssertionInfo: + return AssertionInfo( + description=self.description, + type=AssertionType.VOLUME, + volumeAssertion=VolumeAssertionInfo( + type=VolumeAssertionType.ROW_COUNT_TOTAL, + entity=self.entity, + rowCountTotal=RowCountTotal( + operator=self.operator.operator, + parameters=self.operator.generate_parameters(), + ), + ), + ) + + +class RowCountChangeVolumeAssertion(BaseEntityAssertion): + type: Literal["volume"] + metric: Literal["row_count"] = v1_Field(default="row_count") + change_type: Literal["absolute", "percentage"] + operator: Operators = v1_Field(discriminator="type", alias="condition") + filters: Optional[DatasetFilter] = v1_Field(default=None) + + def get_assertion_info( + self, + ) -> AssertionInfo: + return AssertionInfo( + description=self.description, + type=AssertionType.VOLUME, + volumeAssertion=VolumeAssertionInfo( + type=VolumeAssertionType.ROW_COUNT_CHANGE, + entity=self.entity, + rowCountChange=RowCountChange( + type=( + AssertionValueChangeType.ABSOLUTE + if self.change_type == Literal["absolute"] + else AssertionValueChangeType.PERCENTAGE + ), + operator=self.operator.operator, + parameters=self.operator.generate_parameters(), + ), + ), + ) + + +class VolumeAssertion(BaseAssertionProtocol): + __root__: Union[RowCountTotalVolumeAssertion, RowCountChangeVolumeAssertion] + + @property + def assertion(self): + return self.__root__ + + def get_id(self) -> str: + guid_dict = { + "entity": self.__root__.entity, + "type": self.__root__.type, + "id_raw": self.__root__.id_raw, + } + return self.__root__.id or datahub_guid(guid_dict) + + def get_assertion_info_aspect( + self, + ) -> AssertionInfo: + return self.__root__.get_assertion_info() + + def get_assertion_trigger(self) -> Optional[AssertionTrigger]: + return self.__root__.trigger diff --git a/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py b/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py index 69cbcc4c3e45b..e56e9f059d724 100644 --- a/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py +++ b/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py @@ -16,7 +16,6 @@ OwnershipSourceClass, OwnershipSourceTypeClass, OwnershipTypeClass, - StatusClass, TagAssociationClass, ) from datahub.utilities.urns.data_flow_urn import DataFlowUrn @@ -168,5 +167,5 @@ def generate_data_input_output_mcp( for iolet in self.inlets + self.outlets: yield MetadataChangeProposalWrapper( entityUrn=str(iolet), - aspect=StatusClass(removed=False), + aspect=iolet.to_key_aspect(), ) diff --git a/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py b/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py index fa5b5bd6a50fd..771efd1f2aa51 100644 --- a/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py +++ b/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py @@ -20,7 +20,6 @@ DataProcessInstanceRunResultClass, DataProcessRunStatusClass, DataProcessTypeClass, - StatusClass, ) from datahub.utilities.urns.data_flow_urn import DataFlowUrn from datahub.utilities.urns.data_job_urn import DataJobUrn @@ -107,16 +106,18 @@ def emit_process_start( start_timestamp_millis: int, attempt: Optional[int] = None, emit_template: bool = True, + materialize_iolets: bool = True, callback: Optional[Callable[[Exception, str], None]] = None, ) -> None: """ :rtype: None :param emitter: Datahub Emitter to emit the process event - :param start_timestamp_millis: (int) the execution start time in milliseconds + :param start_timestamp_millis: the execution start time in milliseconds :param attempt: the number of attempt of the execution with the same execution id - :param emit_template: (bool) If it is set the template of the execution (datajob, dataflow) will be emitted as well. - :param callback: (Optional[Callable[[Exception, str], None]]) the callback method for KafkaEmitter if it is used + :param emit_template: If it is set the template of the execution (datajob, dataflow) will be emitted as well. + :param materialize_iolets: If it is set the iolets will be materialized + :param callback: the callback method for KafkaEmitter if it is used """ if emit_template and self.template_urn is not None: template_object: Union[DataJob, DataFlow] @@ -157,7 +158,10 @@ def emit_process_start( for mcp in template_object.generate_mcp(): self._emit_mcp(mcp, emitter, callback) - for mcp in self.generate_mcp(created_ts_millis=start_timestamp_millis): + for mcp in self.generate_mcp( + created_ts_millis=start_timestamp_millis, + materialize_iolets=materialize_iolets, + ): self._emit_mcp(mcp, emitter, callback) for mcp in self.start_event_mcp(start_timestamp_millis, attempt): self._emit_mcp(mcp, emitter, callback) @@ -230,7 +234,7 @@ def emit_process_end( self._emit_mcp(mcp, emitter, callback) def generate_mcp( - self, created_ts_millis: Optional[int] = None, materialize_iolets: bool = True + self, created_ts_millis: Optional[int], materialize_iolets: bool ) -> Iterable[MetadataChangeProposalWrapper]: """Generates mcps from the object""" @@ -280,13 +284,17 @@ def emit( self, emitter: Emitter, callback: Optional[Callable[[Exception, str], None]] = None, + created_ts_millis: Optional[int] = None, ) -> None: """ :param emitter: (Emitter) the datahub emitter to emit generated mcps :param callback: (Optional[Callable[[Exception, str], None]]) the callback method for KafkaEmitter if it is used """ - for mcp in self.generate_mcp(): + for mcp in self.generate_mcp( + created_ts_millis=created_ts_millis, + materialize_iolets=True, + ): self._emit_mcp(mcp, emitter, callback) @staticmethod @@ -363,5 +371,5 @@ def generate_inlet_outlet_mcp( for iolet in self.inlets + self.outlets: yield MetadataChangeProposalWrapper( entityUrn=str(iolet), - aspect=StatusClass(removed=False), + aspect=iolet.to_key_aspect(), ) diff --git a/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py b/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py index 408d6bc7256c6..64b10d31487e0 100644 --- a/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py +++ b/metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py @@ -20,6 +20,8 @@ GlobalTagsClass, GlossaryTermAssociationClass, GlossaryTermsClass, + InstitutionalMemoryClass, + InstitutionalMemoryMetadataClass, KafkaAuditHeaderClass, MetadataChangeProposalClass, OwnerClass, @@ -74,6 +76,15 @@ def ownership_type_must_be_mappable_or_custom(cls, v: str) -> str: return v +class InstitutionMemoryElement(ConfigModel): + url: str + description: str + + +class InstitutionMemory(ConfigModel): + elements: Optional[List[InstitutionMemoryElement]] = None + + class DataProduct(ConfigModel): """This is a DataProduct class which represents a DataProduct @@ -81,6 +92,7 @@ class DataProduct(ConfigModel): id (str): The id of the Data Product domain (str): The domain that the Data Product belongs to. Either as a name or a fully-qualified urn. owners (Optional[List[str, Ownership]]): A list of owners and their types. + institutional_memory (Optional[InstitutionMemory]): A list of institutional memory elements display_name (Optional[str]): The name of the Data Product to display in the UI description (Optional[str]): A documentation string for the Data Product tags (Optional[List[str]]): An array of tags (either bare ids or urns) for the Data Product @@ -94,6 +106,7 @@ class DataProduct(ConfigModel): assets: Optional[List[str]] = None display_name: Optional[str] = None owners: Optional[List[Union[str, Ownership]]] = None + institutional_memory: Optional[InstitutionMemory] = None description: Optional[str] = None tags: Optional[List[str]] = None terms: Optional[List[str]] = None @@ -251,6 +264,22 @@ def generate_mcp( ) yield mcp + if self.institutional_memory and self.institutional_memory.elements: + mcp = MetadataChangeProposalWrapper( + entityUrn=self.urn, + aspect=InstitutionalMemoryClass( + elements=[ + InstitutionalMemoryMetadataClass( + url=element.url, + description=element.description, + createStamp=self._mint_auditstamp("yaml"), + ) + for element in self.institutional_memory.elements + ] + ), + ) + yield mcp + # Finally emit status yield MetadataChangeProposalWrapper( entityUrn=self.urn, aspect=StatusClass(removed=False) diff --git a/metadata-ingestion/src/datahub/api/entities/dataset/dataset.py b/metadata-ingestion/src/datahub/api/entities/dataset/dataset.py index c71bced38f8aa..afeedb83f7998 100644 --- a/metadata-ingestion/src/datahub/api/entities/dataset/dataset.py +++ b/metadata-ingestion/src/datahub/api/entities/dataset/dataset.py @@ -162,6 +162,7 @@ class Dataset(BaseModel): structured_properties: Optional[ Dict[str, Union[str, float, List[Union[str, float]]]] ] = None + external_url: Optional[str] = None @property def platform_urn(self) -> str: @@ -236,6 +237,7 @@ def generate_mcp( description=self.description, name=self.name, customProperties=self.properties, + externalUrl=self.external_url, ), ) yield mcp diff --git a/metadata-ingestion/src/datahub/api/entities/forms/forms.py b/metadata-ingestion/src/datahub/api/entities/forms/forms.py index 5ac08b6e64ed4..8fb7ea0bf11ed 100644 --- a/metadata-ingestion/src/datahub/api/entities/forms/forms.py +++ b/metadata-ingestion/src/datahub/api/entities/forms/forms.py @@ -23,6 +23,7 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.graph.client import DataHubGraph, get_default_graph from datahub.metadata.schema_classes import ( + FormActorAssignmentClass, FormInfoClass, FormPromptClass, OwnerClass, @@ -82,6 +83,12 @@ class Entities(ConfigModel): filters: Optional[Filters] = None +class Actors(ConfigModel): + owners: Optional[bool] = None + users: Optional[List[str]] = None # can be user IDs or urns + groups: Optional[List[str]] = None # can be group IDs or urns + + class Forms(ConfigModel): id: Optional[str] = None urn: Optional[str] = None @@ -93,6 +100,7 @@ class Forms(ConfigModel): entities: Optional[Entities] = None owners: Optional[List[str]] = None # can be user IDs or urns group_owners: Optional[List[str]] = None # can be group IDs or urns + actors: Optional[Actors] = None @validator("urn", pre=True, always=True) def urn_must_be_present(cls, v, values): @@ -124,6 +132,7 @@ def create(file: str) -> None: description=form.description, prompts=form.validate_prompts(emitter), type=form.type, + actors=form.create_form_actors(form.actors), ), ) emitter.emit_mcp(mcp) @@ -201,6 +210,24 @@ def validate_prompts(self, emitter: DataHubGraph) -> List[FormPromptClass]: return prompts + def create_form_actors( + self, actors: Optional[Actors] = None + ) -> Union[None, FormActorAssignmentClass]: + if actors is None: + return None + + users = None + if actors.users is not None: + users = Forms.format_users(actors.users) + + groups = None + if actors.groups is not None: + groups = Forms.format_groups(actors.groups) + + return FormActorAssignmentClass( + owners=actors.owners, users=users, groups=groups + ) + def upload_entities_for_form(self, emitter: DataHubGraph) -> Union[None, Exception]: if self.entities and self.entities.urns: formatted_entity_urns = ", ".join( @@ -260,9 +287,9 @@ def create_form_filters(self, emitter: DataHubGraph) -> Union[None, Exception]: def add_owners(self, emitter: DataHubGraph) -> Union[None, Exception]: owner_urns: List[str] = [] if self.owners: - owner_urns += Forms.format_owners(self.owners) + owner_urns += Forms.format_users(self.owners) if self.group_owners: - owner_urns += Forms.format_group_owners(self.group_owners) + owner_urns += Forms.format_groups(self.group_owners) ownership = OwnershipClass( owners=[ @@ -326,28 +353,28 @@ def from_datahub(graph: DataHubGraph, urn: str) -> "Forms": ) @staticmethod - def format_owners(owners: List[str]) -> List[str]: - formatted_owners: List[str] = [] + def format_users(users: List[str]) -> List[str]: + formatted_users: List[str] = [] - for owner in owners: - if owner.startswith("urn:li:"): - formatted_owners.append(owner) + for user in users: + if user.startswith("urn:li:"): + formatted_users.append(user) else: - formatted_owners.append(make_user_urn(owner)) + formatted_users.append(make_user_urn(user)) - return formatted_owners + return formatted_users @staticmethod - def format_group_owners(owners: List[str]) -> List[str]: - formatted_owners: List[str] = [] + def format_groups(groups: List[str]) -> List[str]: + formatted_groups: List[str] = [] - for owner in owners: - if owner.startswith("urn:li:"): - formatted_owners.append(owner) + for group in groups: + if group.startswith("urn:li:"): + formatted_groups.append(group) else: - formatted_owners.append(make_group_urn(owner)) + formatted_groups.append(make_group_urn(group)) - return formatted_owners + return formatted_groups def to_yaml( self, diff --git a/metadata-ingestion/src/datahub/cli/check_cli.py b/metadata-ingestion/src/datahub/cli/check_cli.py index 4869c568b0967..aec436a25ebe6 100644 --- a/metadata-ingestion/src/datahub/cli/check_cli.py +++ b/metadata-ingestion/src/datahub/cli/check_cli.py @@ -59,10 +59,10 @@ def metadata_file(json_file: str, rewrite: bool, unpack_mces: bool) -> None: "config": {"filename": json_file}, "extractor": "generic", "extractor_config": { - "set_system_metadata": False, "unpack_mces_into_mcps": unpack_mces, }, }, + "flags": {"set_system_metadata": False}, "sink": { "type": "file", "config": {"filename": out_file.name}, diff --git a/metadata-ingestion/src/datahub/cli/cli_utils.py b/metadata-ingestion/src/datahub/cli/cli_utils.py index 1bb3b01e078dd..52aa3cb89d1da 100644 --- a/metadata-ingestion/src/datahub/cli/cli_utils.py +++ b/metadata-ingestion/src/datahub/cli/cli_utils.py @@ -13,11 +13,17 @@ from requests.models import Response from requests.sessions import Session +import datahub from datahub.cli import config_utils from datahub.emitter.aspect import ASPECT_MAP, TIMESERIES_ASPECT_MAP +from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.request_helper import make_curl_command from datahub.emitter.serialization_helper import post_json_transform -from datahub.metadata.schema_classes import _Aspect +from datahub.metadata.com.linkedin.pegasus2avro.mxe import ( + MetadataChangeEvent, + MetadataChangeProposal, +) +from datahub.metadata.schema_classes import SystemMetadataClass, _Aspect from datahub.utilities.urns.urn import Urn, guess_entity_type log = logging.getLogger(__name__) @@ -522,6 +528,7 @@ def get_aspects_for_entity( aspects: List[str], typed: bool = False, cached_session_host: Optional[Tuple[Session, str]] = None, + details: bool = False, ) -> Dict[str, Union[dict, _Aspect]]: # Process non-timeseries aspects non_timeseries_aspects = [a for a in aspects if a not in TIMESERIES_ASPECT_MAP] @@ -553,7 +560,12 @@ def get_aspects_for_entity( aspect_name ) - aspect_dict = a["value"] + if details: + aspect_dict = a + for k in ["name", "version", "type"]: + del aspect_dict[k] + else: + aspect_dict = a["value"] if not typed: aspect_map[aspect_name] = aspect_dict elif aspect_py_class: @@ -683,3 +695,18 @@ def generate_access_token( return token_name, response.json().get("data", {}).get("createAccessToken", {}).get( "accessToken", None ) + + +def ensure_has_system_metadata( + event: Union[ + MetadataChangeProposal, MetadataChangeProposalWrapper, MetadataChangeEvent + ] +) -> None: + if event.systemMetadata is None: + event.systemMetadata = SystemMetadataClass() + metadata = event.systemMetadata + if metadata.properties is None: + metadata.properties = {} + props = metadata.properties + props["clientId"] = datahub.__package_name__ + props["clientVersion"] = datahub.__version__ diff --git a/metadata-ingestion/src/datahub/cli/delete_cli.py b/metadata-ingestion/src/datahub/cli/delete_cli.py index f9e0eb45692d4..9332b701fed39 100644 --- a/metadata-ingestion/src/datahub/cli/delete_cli.py +++ b/metadata-ingestion/src/datahub/cli/delete_cli.py @@ -32,6 +32,7 @@ "domain", "glossaryTerm", "glossaryNode", + "form", } _RECURSIVE_DELETE_TYPES = { diff --git a/metadata-ingestion/src/datahub/cli/get_cli.py b/metadata-ingestion/src/datahub/cli/get_cli.py index 6b779f8565a7f..46e2fdf5b1f79 100644 --- a/metadata-ingestion/src/datahub/cli/get_cli.py +++ b/metadata-ingestion/src/datahub/cli/get_cli.py @@ -21,10 +21,17 @@ def get() -> None: @get.command() @click.option("--urn", required=False, type=str) @click.option("-a", "--aspect", required=False, multiple=True, type=str) +@click.option( + "--details/--no-details", + required=False, + is_flag=True, + default=False, + help="Whether to print details from database which help in audit.", +) @click.pass_context @upgrade.check_upgrade @telemetry.with_telemetry() -def urn(ctx: Any, urn: Optional[str], aspect: List[str]) -> None: +def urn(ctx: Any, urn: Optional[str], aspect: List[str], details: bool) -> None: """ Get metadata for an entity with an optional list of aspects to project. This works for both versioned aspects and timeseries aspects. For timeseries aspects, it fetches the latest value. @@ -39,7 +46,9 @@ def urn(ctx: Any, urn: Optional[str], aspect: List[str]) -> None: logger.debug(f"Using urn from args {urn}") click.echo( json.dumps( - get_aspects_for_entity(entity_urn=urn, aspects=aspect, typed=False), + get_aspects_for_entity( + entity_urn=urn, aspects=aspect, typed=False, details=details + ), sort_keys=True, indent=2, ) diff --git a/metadata-ingestion/src/datahub/cli/ingest_cli.py b/metadata-ingestion/src/datahub/cli/ingest_cli.py index 328a5dc5071c6..bb8d67f8439ab 100644 --- a/metadata-ingestion/src/datahub/cli/ingest_cli.py +++ b/metadata-ingestion/src/datahub/cli/ingest_cli.py @@ -224,7 +224,7 @@ async def run_ingestion_and_check_upgrade() -> int: @click.option( "--urn", type=str, - help="Urn of recipe to update", + help="Urn of recipe to update. Creates recipe if provided urn does not exist", required=False, ) @click.option( @@ -293,10 +293,6 @@ def deploy( variables["schedule"] = {"interval": schedule, "timezone": time_zone} if urn: - if not datahub_graph.exists(urn): - logger.error(f"Could not find recipe for provided urn: {urn}") - exit() - logger.info("Found recipe URN, will update recipe.") graphql_query: str = textwrap.dedent( """ diff --git a/metadata-ingestion/src/datahub/cli/json_file.py b/metadata-ingestion/src/datahub/cli/json_file.py index c3d7a85d722e2..8fb3ea46c64e8 100644 --- a/metadata-ingestion/src/datahub/cli/json_file.py +++ b/metadata-ingestion/src/datahub/cli/json_file.py @@ -9,14 +9,13 @@ def check_mce_file(filepath: str) -> str: mce_source = GenericFileSource.create({"filename": filepath}, None) for _ in mce_source.get_workunits(): pass - if mce_source.get_report().failures: + if len(mce_source.get_report().failures): # raise the first failure found logger.error( f"Event file check failed with errors. Raising first error found. Full report {mce_source.get_report().as_string()}" ) - for failure_list in mce_source.get_report().failures.values(): - if len(failure_list): - raise Exception(failure_list[0]) + for failure in mce_source.get_report().failures: + raise Exception(failure.context) raise Exception( f"Failed to process file due to {mce_source.get_report().failures}" ) diff --git a/metadata-ingestion/src/datahub/cli/specific/assertions_cli.py b/metadata-ingestion/src/datahub/cli/specific/assertions_cli.py new file mode 100644 index 0000000000000..dad724bfe1115 --- /dev/null +++ b/metadata-ingestion/src/datahub/cli/specific/assertions_cli.py @@ -0,0 +1,151 @@ +import logging +import os +from pathlib import Path +from typing import Dict, List, Optional + +import click +from click_default_group import DefaultGroup + +from datahub.api.entities.assertion.assertion_config_spec import AssertionsConfigSpec +from datahub.api.entities.assertion.compiler_interface import ( + AssertionCompilationResult, + CompileResultArtifact, + CompileResultArtifactType, +) +from datahub.emitter.mce_builder import make_assertion_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.graph.client import get_default_graph +from datahub.integrations.assertion.registry import ASSERTION_PLATFORMS +from datahub.telemetry import telemetry +from datahub.upgrade import upgrade + +logger = logging.getLogger(__name__) + +REPORT_FILE_NAME = "compile_report.json" + + +@click.group(cls=DefaultGroup, default="upsert") +def assertions() -> None: + """A group of commands to interact with the Assertion entity in DataHub.""" + pass + + +@assertions.command() +@click.option("-f", "--file", required=True, type=click.Path(exists=True)) +@upgrade.check_upgrade +@telemetry.with_telemetry() +def upsert(file: str) -> None: + """Upsert (create or update) a set of assertions in DataHub.""" + + assertions_spec: AssertionsConfigSpec = AssertionsConfigSpec.from_yaml(file) + + with get_default_graph() as graph: + for assertion_spec in assertions_spec.assertions: + try: + mcp = MetadataChangeProposalWrapper( + entityUrn=make_assertion_urn(assertion_spec.get_id()), + aspect=assertion_spec.get_assertion_info_aspect(), + ) + graph.emit_mcp(mcp) + # TODO: Validate uniqueness of assertion ids. Report if duplicates found. + # TODO: Use upsert graphql endpoints here instead of graph.emit_mcp. + click.secho(f"Update succeeded for urn {mcp.entityUrn}.", fg="green") + except Exception as e: + logger.exception(e) + click.secho( + f"Update failed for {mcp.entityUrn}: {e}", + fg="red", + ) + + +@assertions.command() +@click.option("-f", "--file", required=True, type=click.Path(exists=True)) +@click.option("-p", "--platform", required=True, type=str) +@click.option("-o", "--output-to", required=False, type=click.Path(exists=True)) +@click.option( + "-x", + "--extras", + required=False, + multiple=True, + default=[], + help="Platform-specific extra key-value inputs in form key=value", +) +@upgrade.check_upgrade +@telemetry.with_telemetry() +def compile( + file: str, platform: str, output_to: Optional[str], extras: List[str] +) -> None: + """Compile a set of assertions for input assertion platform. + Note that this does not run any code or execute any queries on assertion platform + and only creates artifacts specific to assertion platform that can be executed manually. + In future, we may introduce separate command to automatically apply these compiled changes + in assertion platform. Currently, generated result artifacts are stored in target folder + unless another folder is specified using option `--output-to `. + """ + + if platform not in ASSERTION_PLATFORMS: + click.secho( + f"Platform {platform} is not supported.", + fg="red", + ) + + if output_to is None: + output_to = f"{os.getcwd()}/target" + + if not os.path.isdir(output_to): + os.mkdir(output_to) + + assertions_spec: AssertionsConfigSpec = AssertionsConfigSpec.from_yaml(file) + + try: + compiler = ASSERTION_PLATFORMS[platform].create( + output_dir=output_to, extras=extras_list_to_dict(extras) + ) + result = compiler.compile(assertions_spec) + + write_report_file(output_to, result) + click.secho("Compile report:", bold=True) + click.echo(result.report.as_string()) + if result.status == "failure": + click.secho("Failure", fg="yellow", bold=True) + else: + click.secho("Success", fg="green", bold=True) + except Exception as e: + logger.exception(e) + click.secho( + f"Compile failed: {e}", + fg="red", + ) + + +def write_report_file(output_to: str, result: AssertionCompilationResult) -> None: + report_path = Path(output_to) / REPORT_FILE_NAME + with (report_path).open("w") as f: + result.add_artifact( + CompileResultArtifact( + name=REPORT_FILE_NAME, + path=report_path, + type=CompileResultArtifactType.COMPILE_REPORT, + description="Detailed report about compile status", + ) + ) + f.write(result.report.as_json()) + + +def extras_list_to_dict(extras: List[str]) -> Dict[str, str]: + extra_properties: Dict[str, str] = dict() + for x in extras: + parts = x.split("=") + assert ( + len(parts) == 2 + ), f"Invalid value for extras {x}, should be in format key=value" + extra_properties[parts[0]] = parts[1] + return extra_properties + + +# TODO: support for +# Immediate: +# 1. delete assertions (from datahub) +# Later: +# 3. execute compiled assertions on assertion platform (Later, requires connection details to platform), +# 4. cleanup assertions from assertion platform (generate artifacts. optionally execute) diff --git a/metadata-ingestion/src/datahub/emitter/mce_builder.py b/metadata-ingestion/src/datahub/emitter/mce_builder.py index ccc1267cd97f7..31c7518152ef2 100644 --- a/metadata-ingestion/src/datahub/emitter/mce_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mce_builder.py @@ -52,6 +52,7 @@ from datahub.utilities.urn_encoder import UrnEncoder from datahub.utilities.urns.data_flow_urn import DataFlowUrn from datahub.utilities.urns.dataset_urn import DatasetUrn +from datahub.utilities.urns.tag_urn import TagUrn logger = logging.getLogger(__name__) Aspect = TypeVar("Aspect", bound=AspectAbstract) @@ -238,8 +239,7 @@ def make_tag_urn(tag: str) -> str: """ if tag and tag.startswith("urn:li:tag:"): return tag - else: - return f"urn:li:tag:{tag}" + return str(TagUrn(tag)) def make_owner_urn(owner: str, owner_type: OwnerType) -> str: @@ -476,7 +476,7 @@ def get_or_add_aspect(mce: MetadataChangeEventClass, default: Aspect) -> Aspect: def make_global_tag_aspect_with_tag_list(tags: List[str]) -> GlobalTagsClass: return GlobalTagsClass( - tags=[TagAssociationClass(f"urn:li:tag:{tag}") for tag in tags] + tags=[TagAssociationClass(make_tag_urn(tag)) for tag in tags] ) diff --git a/metadata-ingestion/src/datahub/emitter/rest_emitter.py b/metadata-ingestion/src/datahub/emitter/rest_emitter.py index 8baa8481ea4f7..bb639339427d9 100644 --- a/metadata-ingestion/src/datahub/emitter/rest_emitter.py +++ b/metadata-ingestion/src/datahub/emitter/rest_emitter.py @@ -10,7 +10,11 @@ from requests.adapters import HTTPAdapter, Retry from requests.exceptions import HTTPError, RequestException -from datahub.cli.cli_utils import fixup_gms_url, get_system_auth +from datahub.cli.cli_utils import ( + ensure_has_system_metadata, + fixup_gms_url, + get_system_auth, +) from datahub.configuration.common import ConfigurationError, OperationalError from datahub.emitter.generic_emitter import Emitter from datahub.emitter.mcp import MetadataChangeProposalWrapper @@ -202,6 +206,7 @@ def emit( UsageAggregation, ], callback: Optional[Callable[[Exception, str], None]] = None, + async_flag: Optional[bool] = None, ) -> None: try: if isinstance(item, UsageAggregation): @@ -209,7 +214,7 @@ def emit( elif isinstance( item, (MetadataChangeProposal, MetadataChangeProposalWrapper) ): - self.emit_mcp(item) + self.emit_mcp(item, async_flag=async_flag) else: self.emit_mce(item) except Exception as e: @@ -228,12 +233,10 @@ def emit_mce(self, mce: MetadataChangeEvent) -> None: snapshot_fqn = ( f"com.linkedin.metadata.snapshot.{mce.proposedSnapshot.RECORD_SCHEMA.name}" ) - system_metadata_obj = {} - if mce.systemMetadata is not None: - system_metadata_obj = { - "lastObserved": mce.systemMetadata.lastObserved, - "runId": mce.systemMetadata.runId, - } + ensure_has_system_metadata(mce) + # To make lint happy + assert mce.systemMetadata is not None + system_metadata_obj = mce.systemMetadata.to_obj() snapshot = { "entity": {"value": {snapshot_fqn: mce_obj}}, "systemMetadata": system_metadata_obj, @@ -243,12 +246,39 @@ def emit_mce(self, mce: MetadataChangeEvent) -> None: self._emit_generic(url, payload) def emit_mcp( - self, mcp: Union[MetadataChangeProposal, MetadataChangeProposalWrapper] + self, + mcp: Union[MetadataChangeProposal, MetadataChangeProposalWrapper], + async_flag: Optional[bool] = None, ) -> None: url = f"{self._gms_server}/aspects?action=ingestProposal" + ensure_has_system_metadata(mcp) mcp_obj = pre_json_transform(mcp.to_obj()) - payload = json.dumps({"proposal": mcp_obj}) + payload_dict = {"proposal": mcp_obj} + + if async_flag is not None: + payload_dict["async"] = "true" if async_flag else "false" + + payload = json.dumps(payload_dict) + + self._emit_generic(url, payload) + + def emit_mcps( + self, + mcps: List[Union[MetadataChangeProposal, MetadataChangeProposalWrapper]], + async_flag: Optional[bool] = None, + ) -> None: + url = f"{self._gms_server}/aspects?action=ingestProposalBatch" + for mcp in mcps: + ensure_has_system_metadata(mcp) + + mcp_objs = [pre_json_transform(mcp.to_obj()) for mcp in mcps] + payload_dict: dict = {"proposals": mcp_objs} + + if async_flag is not None: + payload_dict["async"] = "true" if async_flag else "false" + + payload = json.dumps(payload_dict) self._emit_generic(url, payload) diff --git a/metadata-ingestion/src/datahub/entrypoints.py b/metadata-ingestion/src/datahub/entrypoints.py index 7c5d84b93726d..463bbab496cbb 100644 --- a/metadata-ingestion/src/datahub/entrypoints.py +++ b/metadata-ingestion/src/datahub/entrypoints.py @@ -25,6 +25,7 @@ from datahub.cli.ingest_cli import ingest from datahub.cli.migrate import migrate from datahub.cli.put_cli import put +from datahub.cli.specific.assertions_cli import assertions from datahub.cli.specific.datacontract_cli import datacontract from datahub.cli.specific.dataproduct_cli import dataproduct from datahub.cli.specific.dataset_cli import dataset @@ -36,6 +37,7 @@ from datahub.cli.telemetry import telemetry as telemetry_cli from datahub.cli.timeline_cli import timeline from datahub.configuration.common import should_show_stack_trace +from datahub.ingestion.graph.client import get_default_graph from datahub.telemetry import telemetry from datahub.utilities._custom_package_loader import model_version_name from datahub.utilities.logging_manager import configure_logging @@ -65,7 +67,7 @@ "--log-file", type=click.Path(dir_okay=False), default=None, - help="Enable debug logging.", + help="Write debug-level logs to a file.", ) @click.version_option( version=datahub_package.nice_version_name(), @@ -95,13 +97,23 @@ def datahub( @datahub.command() +@click.option( + "--include-server", + type=bool, + is_flag=True, + default=False, + help="If passed will show server config. Assumes datahub init has happened.", +) @telemetry.with_telemetry() -def version() -> None: +def version(include_server: bool = False) -> None: """Print version number and exit.""" click.echo(f"DataHub CLI version: {datahub_package.nice_version_name()}") click.echo(f"Models: {model_version_name()}") click.echo(f"Python version: {sys.version}") + if include_server: + server_config = get_default_graph().get_config() + click.echo(f"Server config: {server_config}") @datahub.command() @@ -164,6 +176,7 @@ def init(use_password: bool = False) -> None: datahub.add_command(properties) datahub.add_command(forms) datahub.add_command(datacontract) +datahub.add_command(assertions) try: from datahub.cli.lite_cli import lite diff --git a/metadata-ingestion/src/datahub/ingestion/api/source.py b/metadata-ingestion/src/datahub/ingestion/api/source.py index 173f7ec59b0c2..ad1b312ef445c 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/source.py +++ b/metadata-ingestion/src/datahub/ingestion/api/source.py @@ -21,6 +21,7 @@ ) from pydantic import BaseModel +from typing_extensions import LiteralString from datahub.configuration.common import ConfigModel from datahub.configuration.source_common import PlatformInstanceConfigMixin @@ -44,6 +45,8 @@ logger = logging.getLogger(__name__) +_MAX_CONTEXT_STRING_LENGTH = 300 + class SourceCapability(Enum): PLATFORM_INSTANCE = "Platform Instance" @@ -62,6 +65,110 @@ class SourceCapability(Enum): CLASSIFICATION = "Classification" +class StructuredLogLevel(Enum): + INFO = logging.INFO + WARN = logging.WARN + ERROR = logging.ERROR + + +@dataclass +class StructuredLogEntry(Report): + title: Optional[str] + message: str + context: LossyList[str] + + +@dataclass +class StructuredLogs(Report): + # Underlying Lossy Dicts to Capture Errors, Warnings, and Infos. + _entries: Dict[StructuredLogLevel, LossyDict[str, StructuredLogEntry]] = field( + default_factory=lambda: { + StructuredLogLevel.ERROR: LossyDict(10), + StructuredLogLevel.WARN: LossyDict(10), + StructuredLogLevel.INFO: LossyDict(10), + } + ) + + def report_log( + self, + level: StructuredLogLevel, + message: LiteralString, + title: Optional[LiteralString] = None, + context: Optional[str] = None, + exc: Optional[BaseException] = None, + log: bool = False, + ) -> None: + """ + Report a user-facing warning for the ingestion run. + + Args: + level: The level of the log entry. + message: The main message associated with the report entry. This should be a human-readable message. + title: The category / heading to present on for this message in the UI. + context: Additional context (e.g. where, how) for the log entry. + exc: The exception associated with the event. We'll show the stack trace when in debug mode. + """ + + stacklevel = 2 + + log_key = f"{title}-{message}" + entries = self._entries[level] + + if context and len(context) > _MAX_CONTEXT_STRING_LENGTH: + context = f"{context[:_MAX_CONTEXT_STRING_LENGTH]} ..." + + log_content = f"{message} => {context}" if context else message + if exc: + log_content += f"{log_content}: {exc}" + + if log: + logger.log(level=level.value, msg=log_content, stacklevel=stacklevel) + logger.log( + level=logging.DEBUG, + msg="Full stack trace:", + stacklevel=stacklevel, + exc_info=exc, + ) + + # Add the simple exception details to the context. + context = f"{context}: {exc}" + elif log: + logger.log(level=level.value, msg=log_content, stacklevel=stacklevel) + + if log_key not in entries: + context_list: LossyList[str] = LossyList() + if context is not None: + context_list.append(context) + entries[log_key] = StructuredLogEntry( + title=title, + message=message, + context=context_list, + ) + else: + if context is not None: + entries[log_key].context.append(context) + + def _get_of_type(self, level: StructuredLogLevel) -> LossyList[StructuredLogEntry]: + entries = self._entries[level] + result: LossyList[StructuredLogEntry] = LossyList() + for log in entries.values(): + result.append(log) + result.set_total(entries.total_key_count()) + return result + + @property + def warnings(self) -> LossyList[StructuredLogEntry]: + return self._get_of_type(StructuredLogLevel.WARN) + + @property + def failures(self) -> LossyList[StructuredLogEntry]: + return self._get_of_type(StructuredLogLevel.ERROR) + + @property + def infos(self) -> LossyList[StructuredLogEntry]: + return self._get_of_type(StructuredLogLevel.INFO) + + @dataclass class SourceReport(Report): events_produced: int = 0 @@ -76,8 +183,19 @@ class SourceReport(Report): default_factory=lambda: defaultdict(lambda: defaultdict(LossyList)) ) - warnings: LossyDict[str, LossyList[str]] = field(default_factory=LossyDict) - failures: LossyDict[str, LossyList[str]] = field(default_factory=LossyDict) + _structured_logs: StructuredLogs = field(default_factory=StructuredLogs) + + @property + def warnings(self) -> LossyList[StructuredLogEntry]: + return self._structured_logs.warnings + + @property + def failures(self) -> LossyList[StructuredLogEntry]: + return self._structured_logs.failures + + @property + def infos(self) -> LossyList[StructuredLogEntry]: + return self._structured_logs.infos def report_workunit(self, wu: WorkUnit) -> None: self.events_produced += 1 @@ -109,28 +227,74 @@ def report_workunit(self, wu: WorkUnit) -> None: "fineGrainedLineages" ].append(urn) - def report_warning(self, key: str, reason: str) -> None: - warnings = self.warnings.get(key, LossyList()) - warnings.append(reason) - self.warnings[key] = warnings + def report_warning( + self, + message: LiteralString, + context: Optional[str] = None, + title: Optional[LiteralString] = None, + exc: Optional[BaseException] = None, + ) -> None: + self._structured_logs.report_log( + StructuredLogLevel.WARN, message, title, context, exc, log=False + ) - def warning(self, key: str, reason: str) -> None: - self.report_warning(key, reason) - logger.warning(f"{key} => {reason}", stacklevel=2) + def warning( + self, + message: LiteralString, + context: Optional[str] = None, + title: Optional[LiteralString] = None, + exc: Optional[BaseException] = None, + ) -> None: + self._structured_logs.report_log( + StructuredLogLevel.WARN, message, title, context, exc, log=True + ) - def report_failure(self, key: str, reason: str) -> None: - failures = self.failures.get(key, LossyList()) - failures.append(reason) - self.failures[key] = failures + def report_failure( + self, + message: LiteralString, + context: Optional[str] = None, + title: Optional[LiteralString] = None, + exc: Optional[BaseException] = None, + ) -> None: + self._structured_logs.report_log( + StructuredLogLevel.ERROR, message, title, context, exc, log=False + ) - def failure(self, key: str, reason: str) -> None: - self.report_failure(key, reason) - logger.error(f"{key} => {reason}", stacklevel=2) + def failure( + self, + message: LiteralString, + context: Optional[str] = None, + title: Optional[LiteralString] = None, + exc: Optional[BaseException] = None, + ) -> None: + self._structured_logs.report_log( + StructuredLogLevel.ERROR, message, title, context, exc, log=True + ) + + def info( + self, + message: LiteralString, + context: Optional[str] = None, + title: Optional[LiteralString] = None, + exc: Optional[BaseException] = None, + ) -> None: + self._structured_logs.report_log( + StructuredLogLevel.INFO, message, title, context, exc, log=True + ) def __post_init__(self) -> None: self.start_time = datetime.datetime.now() self.running_time: datetime.timedelta = datetime.timedelta(seconds=0) + def as_obj(self) -> dict: + return { + **super().as_obj(), + # To reduce the amount of nesting, we pull these fields out of the structured log. + "failures": Report.to_pure_python_obj(self.failures), + "warnings": Report.to_pure_python_obj(self.warnings), + "infos": Report.to_pure_python_obj(self.infos), + } + def compute_stats(self) -> None: duration = datetime.datetime.now() - self.start_time workunits_produced = self.events_produced diff --git a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py index 7226258515155..372aef707f232 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py +++ b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py @@ -2,7 +2,6 @@ from datetime import datetime, timezone from typing import ( TYPE_CHECKING, - Callable, Dict, Iterable, List, @@ -38,14 +37,12 @@ from datahub.metadata.urns import DatasetUrn, GlossaryTermUrn, TagUrn, Urn from datahub.specific.dataset import DatasetPatchBuilder from datahub.telemetry import telemetry +from datahub.utilities.urns.error import InvalidUrnError from datahub.utilities.urns.urn import guess_entity_type from datahub.utilities.urns.urn_iter import list_urns, lowercase_dataset_urns if TYPE_CHECKING: from datahub.ingestion.api.source import SourceReport - from datahub.ingestion.source.state.stale_entity_removal_handler import ( - StaleEntityRemovalHandler, - ) logger = logging.getLogger(__name__) @@ -102,7 +99,6 @@ def auto_status_aspect( """ all_urns: Set[str] = set() status_urns: Set[str] = set() - skip_urns: Set[str] = set() for wu in stream: urn = wu.get_urn() all_urns.add(urn) @@ -127,53 +123,19 @@ def auto_status_aspect( yield wu - for urn in sorted(all_urns - status_urns - skip_urns): + for urn in sorted(all_urns - status_urns): entity_type = guess_entity_type(urn) if not entity_supports_aspect(entity_type, StatusClass): # If any entity does not support aspect 'status' then skip that entity from adding status aspect. # Example like dataProcessInstance doesn't suppport status aspect. # If not skipped gives error: java.lang.RuntimeException: Unknown aspect status for entity dataProcessInstance continue - yield MetadataChangeProposalWrapper( entityUrn=urn, aspect=StatusClass(removed=False), ).as_workunit() -def _default_entity_type_fn(wu: MetadataWorkUnit) -> Optional[str]: - urn = wu.get_urn() - entity_type = guess_entity_type(urn) - return entity_type - - -def auto_stale_entity_removal( - stale_entity_removal_handler: "StaleEntityRemovalHandler", - stream: Iterable[MetadataWorkUnit], - entity_type_fn: Callable[ - [MetadataWorkUnit], Optional[str] - ] = _default_entity_type_fn, -) -> Iterable[MetadataWorkUnit]: - """ - Record all entities that are found, and emit removals for any that disappeared in this run. - """ - - for wu in stream: - urn = wu.get_urn() - - if wu.is_primary_source: - entity_type = entity_type_fn(wu) - if entity_type is not None: - stale_entity_removal_handler.add_entity_to_state(entity_type, urn) - else: - stale_entity_removal_handler.add_urn_to_skip(urn) - - yield wu - - # Clean up stale entities. - yield from stale_entity_removal_handler.gen_removed_entity_workunits() - - T = TypeVar("T", bound=MetadataWorkUnit) @@ -211,13 +173,18 @@ def auto_materialize_referenced_tags_terms( yield wu for urn in sorted(referenced_tags - tags_with_aspects): - urn_tp = Urn.from_string(urn) - assert isinstance(urn_tp, (TagUrn, GlossaryTermUrn)) + try: + urn_tp = Urn.from_string(urn) + assert isinstance(urn_tp, (TagUrn, GlossaryTermUrn)) - yield MetadataChangeProposalWrapper( - entityUrn=urn, - aspect=urn_tp.to_key_aspect(), - ).as_workunit() + yield MetadataChangeProposalWrapper( + entityUrn=urn, + aspect=urn_tp.to_key_aspect(), + ).as_workunit() + except InvalidUrnError: + logger.info( + f"Source produced an invalid urn, so no key aspect will be generated: {urn}" + ) def auto_lowercase_urns( diff --git a/metadata-ingestion/src/datahub/ingestion/api/workunit.py b/metadata-ingestion/src/datahub/ingestion/api/workunit.py index 82aefda920cb8..f203624aced56 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/workunit.py +++ b/metadata-ingestion/src/datahub/ingestion/api/workunit.py @@ -2,8 +2,6 @@ from dataclasses import dataclass from typing import Iterable, Optional, Type, TypeVar, Union, overload -from deprecated import deprecated - from datahub.emitter.aspect import TIMESERIES_ASPECT_MAP from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import WorkUnit @@ -11,7 +9,7 @@ MetadataChangeEvent, MetadataChangeProposal, ) -from datahub.metadata.schema_classes import UsageAggregationClass, _Aspect +from datahub.metadata.schema_classes import _Aspect logger = logging.getLogger(__name__) @@ -158,11 +156,21 @@ def decompose_mce_into_mcps(self) -> Iterable["MetadataWorkUnit"]: for mcpw in mcps_from_mce(self.metadata) ] - -@deprecated -@dataclass -class UsageStatsWorkUnit(WorkUnit): - usageStats: UsageAggregationClass - - def get_metadata(self) -> dict: - return {"usage": self.usageStats} + @classmethod + def from_metadata( + cls, + metadata: Union[ + MetadataChangeEvent, MetadataChangeProposal, MetadataChangeProposalWrapper + ], + id: Optional[str] = None, + ) -> "MetadataWorkUnit": + workunit_id = id or cls.generate_workunit_id(metadata) + + if isinstance(metadata, MetadataChangeEvent): + return MetadataWorkUnit(id=workunit_id, mce=metadata) + elif isinstance(metadata, (MetadataChangeProposal)): + return MetadataWorkUnit(id=workunit_id, mcp_raw=metadata) + elif isinstance(metadata, MetadataChangeProposalWrapper): + return MetadataWorkUnit(id=workunit_id, mcp=metadata) + else: + raise ValueError(f"Unexpected metadata type {type(metadata)}") diff --git a/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py b/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py index a1fab13fd4bbb..bcf077154343c 100644 --- a/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py +++ b/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py @@ -629,7 +629,7 @@ def get_fields_from_schema( logger.error( "Failed to get fields from schema, continuing...", exc_info=e ) - return [] + return else: raise json_type = cls._get_type_from_schema(jsonref_schema_dict) diff --git a/metadata-ingestion/src/datahub/ingestion/extractor/mce_extractor.py b/metadata-ingestion/src/datahub/ingestion/extractor/mce_extractor.py index 7ad68c0fcf8ea..73e67885dc38f 100644 --- a/metadata-ingestion/src/datahub/ingestion/extractor/mce_extractor.py +++ b/metadata-ingestion/src/datahub/ingestion/extractor/mce_extractor.py @@ -2,15 +2,13 @@ from typing import Iterable, Union from datahub.configuration.common import ConfigModel -from datahub.emitter.mce_builder import get_sys_time from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import RecordEnvelope from datahub.ingestion.api.source import Extractor, WorkUnit -from datahub.ingestion.api.workunit import MetadataWorkUnit, UsageStatsWorkUnit +from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.com.linkedin.pegasus2avro.mxe import ( MetadataChangeEvent, MetadataChangeProposal, - SystemMetadata, ) logger = logging.getLogger(__name__) @@ -26,10 +24,6 @@ def _try_reformat_with_black(code: str) -> str: class WorkUnitRecordExtractorConfig(ConfigModel): - set_system_metadata: bool = True - set_system_metadata_pipeline_name: bool = ( - False # false for now until the models are available in OSS - ) unpack_mces_into_mcps: bool = False @@ -65,14 +59,6 @@ def get_records( MetadataChangeProposalWrapper, ), ): - if self.config.set_system_metadata: - workunit.metadata.systemMetadata = SystemMetadata( - lastObserved=get_sys_time(), runId=self.ctx.run_id - ) - if self.config.set_system_metadata_pipeline_name: - workunit.metadata.systemMetadata.pipelineName = ( - self.ctx.pipeline_name - ) if ( isinstance(workunit.metadata, MetadataChangeEvent) and len(workunit.metadata.proposedSnapshot.aspects) == 0 @@ -92,11 +78,6 @@ def get_records( "workunit_id": workunit.id, }, ) - elif isinstance(workunit, UsageStatsWorkUnit): - logger.error( - "Dropping deprecated `UsageStatsWorkUnit`. " - "Emit a `MetadataWorkUnit` with the `datasetUsageStatistics` aspect instead." - ) else: raise ValueError(f"unknown WorkUnit type {type(workunit)}") diff --git a/metadata-ingestion/src/datahub/ingestion/fs/__init__.py b/metadata-ingestion/src/datahub/ingestion/fs/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/metadata-ingestion/src/datahub/ingestion/fs/fs_base.py b/metadata-ingestion/src/datahub/ingestion/fs/fs_base.py new file mode 100644 index 0000000000000..b099d4d332946 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/fs/fs_base.py @@ -0,0 +1,40 @@ +from abc import ABCMeta, abstractmethod +from dataclasses import dataclass +from typing import Any, Iterable +from urllib import parse + + +@dataclass +class FileInfo: + path: str + size: int + is_file: bool + + def __str__(self): + return f"FileInfo({self.path}, {self.size}, {self.is_file})" + + +class FileSystem(metaclass=ABCMeta): + @classmethod + def create(cls, **kwargs: Any) -> "FileSystem": + raise NotImplementedError('File system implementations must implement "create"') + + @abstractmethod + def open(self, path: str, **kwargs: Any) -> Any: + pass + + @abstractmethod + def file_status(self, path: str) -> FileInfo: + pass + + @abstractmethod + def list(self, path: str) -> Iterable[FileInfo]: + pass + + +def get_path_schema(path: str) -> str: + scheme = parse.urlparse(path).scheme + if scheme == "": + # This makes the default schema "file" for local paths. + scheme = "file" + return scheme diff --git a/metadata-ingestion/src/datahub/ingestion/fs/fs_registry.py b/metadata-ingestion/src/datahub/ingestion/fs/fs_registry.py new file mode 100644 index 0000000000000..cb2349723a4cd --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/fs/fs_registry.py @@ -0,0 +1,5 @@ +from datahub.ingestion.api.registry import PluginRegistry +from datahub.ingestion.fs.fs_base import FileSystem + +fs_registry = PluginRegistry[FileSystem]() +fs_registry.register_from_entrypoint("datahub.fs.plugins") diff --git a/metadata-ingestion/src/datahub/ingestion/fs/http_fs.py b/metadata-ingestion/src/datahub/ingestion/fs/http_fs.py new file mode 100644 index 0000000000000..a9153352690ad --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/fs/http_fs.py @@ -0,0 +1,28 @@ +from typing import Any, Iterable + +import requests +import smart_open + +from datahub.ingestion.fs.fs_base import FileInfo, FileSystem + + +class HttpFileSystem(FileSystem): + @classmethod + def create(cls, **kwargs): + return HttpFileSystem() + + def open(self, path: str, **kwargs: Any) -> Any: + return smart_open.open(path, mode="rb", transport_params=kwargs) + + def file_status(self, path: str) -> FileInfo: + head = requests.head(path) + if head.ok: + return FileInfo(path, int(head.headers["Content-length"]), is_file=True) + elif head.status_code == 404: + raise FileNotFoundError(f"Requested path {path} does not exist.") + else: + raise IOError(f"Cannot get file status for the requested path {path}.") + + def list(self, path: str) -> Iterable[FileInfo]: + status = self.file_status(path) + return [status] diff --git a/metadata-ingestion/src/datahub/ingestion/fs/local_fs.py b/metadata-ingestion/src/datahub/ingestion/fs/local_fs.py new file mode 100644 index 0000000000000..8a546650a3dfe --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/fs/local_fs.py @@ -0,0 +1,29 @@ +import os +import pathlib +from typing import Any, Iterable + +from datahub.ingestion.fs.fs_base import FileInfo, FileSystem + + +class LocalFileSystem(FileSystem): + @classmethod + def create(cls, **kwargs): + return LocalFileSystem() + + def open(self, path: str, **kwargs: Any) -> Any: + # Local does not support any additional kwargs + assert not kwargs + return pathlib.Path(path).open(mode="rb") + + def list(self, path: str) -> Iterable[FileInfo]: + p = pathlib.Path(path) + if p.is_file(): + return [self.file_status(path)] + else: + return iter([self.file_status(str(x)) for x in p.iterdir()]) + + def file_status(self, path: str) -> FileInfo: + if os.path.isfile(path): + return FileInfo(path, os.path.getsize(path), is_file=True) + else: + return FileInfo(path, 0, is_file=False) diff --git a/metadata-ingestion/src/datahub/ingestion/fs/s3_fs.py b/metadata-ingestion/src/datahub/ingestion/fs/s3_fs.py new file mode 100644 index 0000000000000..a135b7b6ce837 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/fs/s3_fs.py @@ -0,0 +1,108 @@ +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Any, Iterable +from urllib.parse import urlparse + +import boto3 +import smart_open + +from datahub.ingestion.fs import s3_fs +from datahub.ingestion.fs.fs_base import FileInfo, FileSystem + + +def parse_s3_path(path: str) -> "S3Path": + parsed = urlparse(path) + return S3Path(parsed.netloc, parsed.path.lstrip("/")) + + +def assert_ok_status(s3_response): + is_ok = s3_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert ( + is_ok + ), f"Failed to fetch S3 object, error message: {s3_response['Error']['Message']}" + + +@dataclass +class S3Path: + bucket: str + key: str + + def __str__(self): + return f"S3Path({self.bucket}, {self.key})" + + +class S3ListIterator(Iterator): + + MAX_KEYS = 1000 + + def __init__( + self, s3_client: Any, bucket: str, prefix: str, max_keys: int = MAX_KEYS + ) -> None: + self._s3 = s3_client + self._bucket = bucket + self._prefix = prefix + self._max_keys = max_keys + self._file_statuses: Iterator = iter([]) + self._token = "" + self.fetch() + + def __next__(self) -> FileInfo: + try: + return next(self._file_statuses) + except StopIteration: + if self._token: + self.fetch() + return next(self._file_statuses) + else: + raise StopIteration() + + def fetch(self): + params = dict(Bucket=self._bucket, Prefix=self._prefix, MaxKeys=self._max_keys) + if self._token: + params.update(ContinuationToken=self._token) + + response = self._s3.list_objects_v2(**params) + + s3_fs.assert_ok_status(response) + + self._file_statuses = iter( + [ + FileInfo(f"s3://{response['Name']}/{x['Key']}", x["Size"], is_file=True) + for x in response.get("Contents", []) + ] + ) + self._token = response.get("NextContinuationToken") + + +class S3FileSystem(FileSystem): + def __init__(self, **kwargs): + self.s3 = boto3.client("s3", **kwargs) + + @classmethod + def create(cls, **kwargs): + return S3FileSystem(**kwargs) + + def open(self, path: str, **kwargs: Any) -> Any: + transport_params = kwargs.update({"client": self.s3}) + return smart_open.open(path, mode="rb", transport_params=transport_params) + + def file_status(self, path: str) -> FileInfo: + s3_path = parse_s3_path(path) + try: + response = self.s3.get_object_attributes( + Bucket=s3_path.bucket, Key=s3_path.key, ObjectAttributes=["ObjectSize"] + ) + assert_ok_status(response) + return FileInfo(path, response["ObjectSize"], is_file=True) + except Exception as e: + if ( + hasattr(e, "response") + and e.response["ResponseMetadata"]["HTTPStatusCode"] == 404 + ): + return FileInfo(path, 0, is_file=False) + else: + raise e + + def list(self, path: str) -> Iterable[FileInfo]: + s3_path = parse_s3_path(path) + return S3ListIterator(self.s3, s3_path.bucket, s3_path.key) diff --git a/metadata-ingestion/src/datahub/ingestion/glossary/classification_mixin.py b/metadata-ingestion/src/datahub/ingestion/glossary/classification_mixin.py index d4b649a637ffb..3e5eb4347b474 100644 --- a/metadata-ingestion/src/datahub/ingestion/glossary/classification_mixin.py +++ b/metadata-ingestion/src/datahub/ingestion/glossary/classification_mixin.py @@ -300,6 +300,8 @@ def classification_workunit_processor( table_name = ".".join(table_id) if not classification_handler.is_classification_enabled_for_table(table_name): yield from table_wu_generator + return + for wu in table_wu_generator: maybe_schema_metadata = wu.get_aspect_of_type(SchemaMetadata) if ( diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py index 887d7ec703c0a..7ba412b3e772c 100644 --- a/metadata-ingestion/src/datahub/ingestion/graph/client.py +++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py @@ -15,6 +15,7 @@ Iterable, Iterator, List, + Literal, Optional, Tuple, Type, @@ -213,7 +214,7 @@ def _post_generic(self, url: str, payload_dict: Dict) -> Dict: def _make_rest_sink_config(self) -> "DatahubRestSinkConfig": from datahub.ingestion.sink.datahub_rest import ( DatahubRestSinkConfig, - SyncOrAsync, + RestSinkMode, ) # This is a bit convoluted - this DataHubGraph class is a subclass of DatahubRestEmitter, @@ -221,7 +222,7 @@ def _make_rest_sink_config(self) -> "DatahubRestSinkConfig": # TODO: We should refactor out the multithreading functionality of the sink # into a separate class that can be used by both the sink and the graph client # e.g. a DatahubBulkRestEmitter that both the sink and the graph client use. - return DatahubRestSinkConfig(**self.config.dict(), mode=SyncOrAsync.ASYNC) + return DatahubRestSinkConfig(**self.config.dict(), mode=RestSinkMode.ASYNC) @contextlib.contextmanager def make_rest_sink( @@ -252,14 +253,10 @@ def emit_all( ) -> None: """Emit all items in the iterable using multiple threads.""" + # The context manager also ensures that we raise an error if a failure occurs. with self.make_rest_sink(run_id=run_id) as sink: for item in items: sink.emit_async(item) - if sink.report.failures: - raise OperationalError( - f"Failed to emit {len(sink.report.failures)} records", - info=sink.report.as_obj(), - ) def get_aspect( self, @@ -1281,6 +1278,329 @@ def create_tag(self, tag_name: str) -> str: # return urn return res["createTag"] + def remove_tag(self, tag_urn: str, resource_urn: str) -> bool: + graph_query = f""" + mutation removeTag {{ + removeTag( + input: {{ + tagUrn: "{tag_urn}", + resourceUrn: "{resource_urn}" + }}) + }} + """ + + res = self.execute_graphql(query=graph_query) + return res["removeTag"] + + def _assertion_result_shared(self) -> str: + fragment: str = """ + fragment assertionResult on AssertionResult { + type + rowCount + missingCount + unexpectedCount + actualAggValue + externalUrl + nativeResults { + value + } + error { + type + properties { + value + } + } + } + """ + return fragment + + def _run_assertion_result_shared(self) -> str: + fragment: str = """ + fragment runAssertionResult on RunAssertionResult { + assertion { + urn + } + result { + ... assertionResult + } + } + """ + return fragment + + def _run_assertion_build_params( + self, params: Optional[Dict[str, str]] = {} + ) -> List[Any]: + if params is None: + return [] + + results = [] + for key, value in params.items(): + result = { + "key": key, + "value": value, + } + results.append(result) + + return results + + def run_assertion( + self, + urn: str, + save_result: bool = True, + parameters: Optional[Dict[str, str]] = {}, + async_flag: bool = False, + ) -> Dict: + params = self._run_assertion_build_params(parameters) + graph_query: str = """ + %s + mutation runAssertion($assertionUrn: String!, $saveResult: Boolean, $parameters: [StringMapEntryInput!], $async: Boolean!) { + runAssertion(urn: $assertionUrn, saveResult: $saveResult, parameters: $parameters, async: $async) { + ... assertionResult + } + } + """ % ( + self._assertion_result_shared() + ) + + variables = { + "assertionUrn": urn, + "saveResult": save_result, + "parameters": params, + "async": async_flag, + } + + res = self.execute_graphql( + query=graph_query, + variables=variables, + ) + + return res["runAssertion"] + + def run_assertions( + self, + urns: List[str], + save_result: bool = True, + parameters: Optional[Dict[str, str]] = {}, + async_flag: bool = False, + ) -> Dict: + params = self._run_assertion_build_params(parameters) + graph_query: str = """ + %s + %s + mutation runAssertions($assertionUrns: [String!]!, $saveResult: Boolean, $parameters: [StringMapEntryInput!], $async: Boolean!) { + runAssertions(urns: $assertionUrns, saveResults: $saveResult, parameters: $parameters, async: $async) { + passingCount + failingCount + errorCount + results { + ... runAssertionResult + } + } + } + """ % ( + self._assertion_result_shared(), + self._run_assertion_result_shared(), + ) + + variables = { + "assertionUrns": urns, + "saveResult": save_result, + "parameters": params, + "async": async_flag, + } + + res = self.execute_graphql( + query=graph_query, + variables=variables, + ) + + return res["runAssertions"] + + def run_assertions_for_asset( + self, + urn: str, + tag_urns: Optional[List[str]] = [], + parameters: Optional[Dict[str, str]] = {}, + async_flag: bool = False, + ) -> Dict: + params = self._run_assertion_build_params(parameters) + graph_query: str = """ + %s + %s + mutation runAssertionsForAsset($assetUrn: String!, $tagUrns: [String!], $parameters: [StringMapEntryInput!], $async: Boolean!) { + runAssertionsForAsset(urn: $assetUrn, tagUrns: $tagUrns, parameters: $parameters, async: $async) { + passingCount + failingCount + errorCount + results { + ... runAssertionResult + } + } + } + """ % ( + self._assertion_result_shared(), + self._run_assertion_result_shared(), + ) + + variables = { + "assetUrn": urn, + "tagUrns": tag_urns, + "parameters": params, + "async": async_flag, + } + + res = self.execute_graphql( + query=graph_query, + variables=variables, + ) + + return res["runAssertionsForAsset"] + + def get_entities_v2( + self, + entity_name: str, + urns: List[str], + aspects: List[str] = [], + with_system_metadata: bool = False, + ) -> Dict[str, Any]: + payload = { + "urns": urns, + "aspectNames": aspects, + "withSystemMetadata": with_system_metadata, + } + headers: Dict[str, Any] = { + "Accept": "application/json", + "Content-Type": "application/json", + } + url = f"{self.config.server}/openapi/v2/entity/batch/{entity_name}" + response = self._session.post(url, data=json.dumps(payload), headers=headers) + response.raise_for_status() + + json_resp = response.json() + entities = json_resp.get("entities", []) + aspects_set = set(aspects) + retval: Dict[str, Any] = {} + + for entity in entities: + entity_aspects = entity.get("aspects", {}) + entity_urn = entity.get("urn", None) + + if entity_urn is None: + continue + for aspect_key, aspect_value in entity_aspects.items(): + # Include all aspects if aspect filter is empty + if len(aspects) == 0 or aspect_key in aspects_set: + retval.setdefault(entity_urn, {}) + retval[entity_urn][aspect_key] = aspect_value + return retval + + def upsert_custom_assertion( + self, + urn: Optional[str], + entity_urn: str, + type: str, + description: str, + platform_name: Optional[str] = None, + platform_urn: Optional[str] = None, + field_path: Optional[str] = None, + external_url: Optional[str] = None, + logic: Optional[str] = None, + ) -> Dict: + graph_query: str = """ + mutation upsertCustomAssertion( + $assertionUrn: String, + $entityUrn: String!, + $type: String!, + $description: String!, + $fieldPath: String, + $platformName: String, + $platformUrn: String, + $externalUrl: String, + $logic: String + ) { + upsertCustomAssertion(urn: $assertionUrn, input: { + entityUrn: $entityUrn + type: $type + description: $description + fieldPath: $fieldPath + platform: { + urn: $platformUrn + name: $platformName + } + externalUrl: $externalUrl + logic: $logic + }) { + urn + } + } + """ + + variables = { + "assertionUrn": urn, + "entityUrn": entity_urn, + "type": type, + "description": description, + "fieldPath": field_path, + "platformName": platform_name, + "platformUrn": platform_urn, + "externalUrl": external_url, + "logic": logic, + } + + res = self.execute_graphql( + query=graph_query, + variables=variables, + ) + + return res["upsertCustomAssertion"] + + def report_assertion_result( + self, + urn: str, + timestamp_millis: int, + type: Literal["SUCCESS", "FAILURE", "ERROR", "INIT"], + properties: Optional[List[Dict[str, str]]] = None, + external_url: Optional[str] = None, + error_type: Optional[str] = None, + error_message: Optional[str] = None, + ) -> bool: + graph_query: str = """ + mutation reportAssertionResult( + $assertionUrn: String!, + $timestampMillis: Long!, + $type: AssertionResultType!, + $properties: [StringMapEntryInput!], + $externalUrl: String, + $error: AssertionResultErrorInput, + ) { + reportAssertionResult(urn: $assertionUrn, result: { + timestampMillis: $timestampMillis + type: $type + properties: $properties + externalUrl: $externalUrl + error: $error + }) + } + """ + + variables = { + "assertionUrn": urn, + "timestampMillis": timestamp_millis, + "type": type, + "properties": properties, + "externalUrl": external_url, + "error": {"type": error_type, "message": error_message} + if error_type + else None, + } + + res = self.execute_graphql( + query=graph_query, + variables=variables, + ) + + return res["reportAssertionResult"] + def close(self) -> None: self._make_schema_resolver.cache_clear() super().close() diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py index 75dabc4a7e02a..e61ffa46b3c10 100644 --- a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py +++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py @@ -1,4 +1,5 @@ import contextlib +import enum import itertools import logging import os @@ -38,6 +39,9 @@ from datahub.ingestion.sink.file import FileSink, FileSinkConfig from datahub.ingestion.sink.sink_registry import sink_registry from datahub.ingestion.source.source_registry import source_registry +from datahub.ingestion.transformer.system_metadata_transformer import ( + SystemMetadataTransformer, +) from datahub.ingestion.transformer.transform_registry import transform_registry from datahub.metadata.schema_classes import MetadataChangeProposalClass from datahub.telemetry import stats, telemetry @@ -46,9 +50,10 @@ clear_global_warnings, get_global_warnings, ) -from datahub.utilities.lossy_collections import LossyDict, LossyList +from datahub.utilities.lossy_collections import LossyList logger = logging.getLogger(__name__) +_REPORT_PRINT_INTERVAL_SECONDS = 60 class LoggingCallback(WriteCallback): @@ -116,6 +121,13 @@ class PipelineInitError(Exception): pass +class PipelineStatus(enum.Enum): + UNKNOWN = enum.auto() + COMPLETED = enum.auto() + ERROR = enum.auto() + CANCELLED = enum.auto() + + @contextlib.contextmanager def _add_init_error_context(step: str) -> Iterator[None]: """Enriches any exceptions raised with information about the step that failed.""" @@ -308,6 +320,9 @@ def _configure_transforms(self) -> None: f"Transformer type:{transformer_type},{transformer_class} configured" ) + # Add the system metadata transformer at the end of the list. + self.transformers.append(SystemMetadataTransformer(self.ctx)) + def _configure_reporting( self, report_to: Optional[str], no_default_report: bool ) -> None: @@ -365,11 +380,11 @@ def _notify_reporters_on_ingestion_completion(self) -> None: try: reporter.on_completion( status="CANCELLED" - if self.final_status == "cancelled" + if self.final_status == PipelineStatus.CANCELLED else "FAILURE" if self.has_failures() else "SUCCESS" - if self.final_status == "completed" + if self.final_status == PipelineStatus.COMPLETED else "UNKNOWN", report=self._get_structured_report(), ctx=self.ctx, @@ -403,7 +418,7 @@ def create( def _time_to_print(self) -> bool: self.num_intermediate_workunits += 1 current_time = int(time.time()) - if current_time - self.last_time_printed > 10: + if current_time - self.last_time_printed > _REPORT_PRINT_INTERVAL_SECONDS: # we print self.num_intermediate_workunits = 0 self.last_time_printed = current_time @@ -421,7 +436,7 @@ def run(self) -> None: ) ) - self.final_status = "unknown" + self.final_status = PipelineStatus.UNKNOWN self._notify_reporters_on_ingestion_start() callback = None try: @@ -458,9 +473,7 @@ def run(self) -> None: f"Failed to write record: {e}" ) - except RuntimeError: - raise - except SystemExit: + except (RuntimeError, SystemExit): raise except Exception as e: logger.error( @@ -489,11 +502,14 @@ def run(self) -> None: self.sink.write_record_async(record_envelope, callback) self.process_commits() - self.final_status = "completed" - except (SystemExit, RuntimeError, KeyboardInterrupt) as e: - self.final_status = "cancelled" + self.final_status = PipelineStatus.COMPLETED + except (SystemExit, KeyboardInterrupt) as e: + self.final_status = PipelineStatus.CANCELLED logger.error("Caught error", exc_info=e) raise + except Exception as exc: + self.final_status = PipelineStatus.ERROR + self._handle_uncaught_pipeline_exception(exc) finally: clear_global_warnings() @@ -603,11 +619,8 @@ def log_ingestion_stats(self) -> None: self.ctx.graph, ) - def _approx_all_vals(self, d: LossyDict[str, LossyList]) -> int: - result = d.dropped_keys_count() - for k in d: - result += len(d[k]) - return result + def _approx_all_vals(self, d: LossyList[Any]) -> int: + return d.total_elements def _get_text_color(self, running: bool, failures: bool, warnings: bool) -> str: if running: @@ -628,20 +641,30 @@ def has_failures(self) -> bool: def pretty_print_summary( self, warnings_as_failure: bool = False, currently_running: bool = False ) -> int: - click.echo() - click.secho("Cli report:", bold=True) - click.secho(self.cli_report.as_string()) - click.secho(f"Source ({self.source_type}) report:", bold=True) - click.echo(self.source.get_report().as_string()) - click.secho(f"Sink ({self.sink_type}) report:", bold=True) - click.echo(self.sink.get_report().as_string()) - global_warnings = get_global_warnings() - if len(global_warnings) > 0: - click.secho("Global Warnings:", bold=True) - click.echo(global_warnings) - click.echo() workunits_produced = self.sink.get_report().total_records_written + if ( + not workunits_produced + and not currently_running + and self.final_status == PipelineStatus.ERROR + ): + # If the pipeline threw an uncaught exception before doing anything, printing + # out the report would just be annoying. + pass + else: + click.echo() + click.secho("Cli report:", bold=True) + click.secho(self.cli_report.as_string()) + click.secho(f"Source ({self.source_type}) report:", bold=True) + click.echo(self.source.get_report().as_string()) + click.secho(f"Sink ({self.sink_type}) report:", bold=True) + click.echo(self.sink.get_report().as_string()) + global_warnings = get_global_warnings() + if len(global_warnings) > 0: + click.secho("Global Warnings:", bold=True) + click.echo(global_warnings) + click.echo() + duration_message = f"in {humanfriendly.format_timespan(self.source.get_report().running_time)}." if currently_running: message_template = f"⏳ Pipeline running {{status}} so far; produced {workunits_produced} events {duration_message}" @@ -691,6 +714,14 @@ def pretty_print_summary( ) return 0 + def _handle_uncaught_pipeline_exception(self, exc: Exception) -> None: + logger.exception("Ingestion pipeline threw an uncaught exception") + self.source.get_report().report_failure( + title="Pipeline Error", + message="Ingestion pipeline raised an unexpected exception!", + exc=exc, + ) + def _get_structured_report(self) -> Dict[str, Any]: return { "cli": self.cli_report.as_obj(), diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py index 179a1df3a5226..98629ba030695 100644 --- a/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py +++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py @@ -62,6 +62,14 @@ class FlagsConfig(ConfigModel): ), ) + set_system_metadata: bool = Field( + True, description="Set system metadata on entities." + ) + set_system_metadata_pipeline_name: bool = Field( + True, + description="Set system metadata pipeline name. Requires `set_system_metadata` to be enabled.", + ) + class PipelineConfig(ConfigModel): source: SourceConfig diff --git a/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py b/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py index 007b7487cb6a4..33a8f4a126182 100644 --- a/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py +++ b/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py @@ -1,11 +1,13 @@ import concurrent.futures import contextlib +import dataclasses import functools import logging +import os +import threading import uuid -from dataclasses import dataclass from enum import auto -from typing import Optional, Union +from typing import List, Optional, Tuple, Union from datahub.cli.cli_utils import set_env_variables_override_config from datahub.configuration.common import ( @@ -14,7 +16,8 @@ OperationalError, ) from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.emitter.rest_emitter import DatahubRestEmitter +from datahub.emitter.mcp_builder import mcps_from_mce +from datahub.emitter.rest_emitter import DataHubRestEmitter from datahub.ingestion.api.common import RecordEnvelope, WorkUnit from datahub.ingestion.api.sink import ( NoopWriteCallback, @@ -28,31 +31,49 @@ MetadataChangeEvent, MetadataChangeProposal, ) -from datahub.utilities.advanced_thread_executor import PartitionExecutor +from datahub.utilities.partition_executor import ( + BatchPartitionExecutor, + PartitionExecutor, +) +from datahub.utilities.perf_timer import PerfTimer from datahub.utilities.server_config_util import set_gms_config logger = logging.getLogger(__name__) +DEFAULT_REST_SINK_MAX_THREADS = int( + os.getenv("DATAHUB_REST_SINK_DEFAULT_MAX_THREADS", 15) +) + -class SyncOrAsync(ConfigEnum): +class RestSinkMode(ConfigEnum): SYNC = auto() ASYNC = auto() + # Uses the new ingestProposalBatch endpoint. Significantly more efficient than the other modes, + # but requires a server version that supports it. + # https://github.com/datahub-project/datahub/pull/10706 + ASYNC_BATCH = auto() + class DatahubRestSinkConfig(DatahubClientConfig): - mode: SyncOrAsync = SyncOrAsync.ASYNC + mode: RestSinkMode = RestSinkMode.ASYNC + + # These only apply in async modes. + max_threads: int = DEFAULT_REST_SINK_MAX_THREADS + max_pending_requests: int = 2000 - # These only apply in async mode. - max_threads: int = 15 - max_pending_requests: int = 500 + # Only applies in async batch mode. + max_per_batch: int = 100 -@dataclass +@dataclasses.dataclass class DataHubRestSinkReport(SinkReport): - max_threads: int = -1 - gms_version: str = "" + max_threads: Optional[int] = None + gms_version: Optional[str] = None pending_requests: int = 0 + main_thread_blocking_timer: PerfTimer = dataclasses.field(default_factory=PerfTimer) + def compute_stats(self) -> None: super().compute_stats() @@ -79,22 +100,12 @@ def _get_partition_key(record_envelope: RecordEnvelope) -> str: class DatahubRestSink(Sink[DatahubRestSinkConfig, DataHubRestSinkReport]): - emitter: DatahubRestEmitter + _emitter_thread_local: threading.local treat_errors_as_warnings: bool = False def __post_init__(self) -> None: - self.emitter = DatahubRestEmitter( - self.config.server, - self.config.token, - connect_timeout_sec=self.config.timeout_sec, # reuse timeout_sec for connect timeout - read_timeout_sec=self.config.timeout_sec, - retry_status_codes=self.config.retry_status_codes, - retry_max_times=self.config.retry_max_times, - extra_headers=self.config.extra_headers, - ca_certificate_path=self.config.ca_certificate_path, - client_certificate_path=self.config.client_certificate_path, - disable_ssl_verification=self.config.disable_ssl_verification, - ) + self._emitter_thread_local = threading.local() + try: gms_config = self.emitter.get_server_config() except Exception as exc: @@ -105,18 +116,54 @@ def __post_init__(self) -> None: self.report.gms_version = ( gms_config.get("versions", {}) .get("acryldata/datahub", {}) - .get("version", "") + .get("version", None) ) self.report.max_threads = self.config.max_threads logger.debug("Setting env variables to override config") set_env_variables_override_config(self.config.server, self.config.token) logger.debug("Setting gms config") set_gms_config(gms_config) - self.executor = PartitionExecutor( - max_workers=self.config.max_threads, - max_pending=self.config.max_pending_requests, + + self.executor: Union[PartitionExecutor, BatchPartitionExecutor] + if self.config.mode == RestSinkMode.ASYNC_BATCH: + self.executor = BatchPartitionExecutor( + max_workers=self.config.max_threads, + max_pending=self.config.max_pending_requests, + process_batch=self._emit_batch_wrapper, + max_per_batch=self.config.max_per_batch, + ) + else: + self.executor = PartitionExecutor( + max_workers=self.config.max_threads, + max_pending=self.config.max_pending_requests, + ) + + @classmethod + def _make_emitter(cls, config: DatahubRestSinkConfig) -> DataHubRestEmitter: + return DataHubRestEmitter( + config.server, + config.token, + connect_timeout_sec=config.timeout_sec, # reuse timeout_sec for connect timeout + read_timeout_sec=config.timeout_sec, + retry_status_codes=config.retry_status_codes, + retry_max_times=config.retry_max_times, + extra_headers=config.extra_headers, + ca_certificate_path=config.ca_certificate_path, + client_certificate_path=config.client_certificate_path, + disable_ssl_verification=config.disable_ssl_verification, ) + @property + def emitter(self) -> DataHubRestEmitter: + # While this is a property, it actually uses one emitter per thread. + # Since emitter is one-to-one with request sessions, using a separate + # emitter per thread should improve correctness and performance. + # https://github.com/psf/requests/issues/1871#issuecomment-32751346 + thread_local = self._emitter_thread_local + if not hasattr(thread_local, "emitter"): + thread_local.emitter = DatahubRestSink._make_emitter(self.config) + return thread_local.emitter + def handle_work_unit_start(self, workunit: WorkUnit) -> None: if isinstance(workunit, MetadataWorkUnit): self.treat_errors_as_warnings = workunit.treat_errors_as_warnings @@ -164,6 +211,7 @@ def _write_done_callback( self.report.report_warning({"warning": e.message, "info": e.info}) write_callback.on_failure(record_envelope, e, e.info) else: + logger.exception(f"Failure: {e}", exc_info=e) self.report.report_failure({"e": e}) write_callback.on_failure(record_envelope, Exception(e), {}) @@ -178,6 +226,30 @@ def _emit_wrapper( # TODO: Add timing metrics self.emitter.emit(record) + def _emit_batch_wrapper( + self, + records: List[ + Tuple[ + Union[ + MetadataChangeEvent, + MetadataChangeProposal, + MetadataChangeProposalWrapper, + ], + ] + ], + ) -> None: + events: List[Union[MetadataChangeProposal, MetadataChangeProposalWrapper]] = [] + for record in records: + event = record[0] + if isinstance(event, MetadataChangeEvent): + # Unpack MCEs into MCPs. + mcps = mcps_from_mce(event) + events.extend(mcps) + else: + events.append(event) + + self.emitter.emit_mcps(events) + def write_record_async( self, record_envelope: RecordEnvelope[ @@ -189,25 +261,40 @@ def write_record_async( ], write_callback: WriteCallback, ) -> None: - record = record_envelope.record - if self.config.mode == SyncOrAsync.ASYNC: - partition_key = _get_partition_key(record_envelope) - self.executor.submit( - partition_key, - self._emit_wrapper, - record, - done_callback=functools.partial( - self._write_done_callback, record_envelope, write_callback - ), - ) - self.report.pending_requests += 1 - else: - # execute synchronously - try: - self._emit_wrapper(record) - write_callback.on_success(record_envelope, success_metadata={}) - except Exception as e: - write_callback.on_failure(record_envelope, e, failure_metadata={}) + # Because the default is async mode and most sources are slower than the sink, this + # should only have a high value if the sink is actually a bottleneck. + with self.report.main_thread_blocking_timer: + record = record_envelope.record + if self.config.mode == RestSinkMode.ASYNC: + assert isinstance(self.executor, PartitionExecutor) + partition_key = _get_partition_key(record_envelope) + self.executor.submit( + partition_key, + self._emit_wrapper, + record, + done_callback=functools.partial( + self._write_done_callback, record_envelope, write_callback + ), + ) + self.report.pending_requests += 1 + elif self.config.mode == RestSinkMode.ASYNC_BATCH: + assert isinstance(self.executor, BatchPartitionExecutor) + partition_key = _get_partition_key(record_envelope) + self.executor.submit( + partition_key, + record, + done_callback=functools.partial( + self._write_done_callback, record_envelope, write_callback + ), + ) + self.report.pending_requests += 1 + else: + # execute synchronously + try: + self._emit_wrapper(record) + write_callback.on_success(record_envelope, success_metadata={}) + except Exception as e: + write_callback.on_failure(record_envelope, e, failure_metadata={}) def emit_async( self, @@ -221,7 +308,8 @@ def emit_async( ) def close(self): - self.executor.shutdown() + with self.report.main_thread_blocking_timer: + self.executor.shutdown() def __repr__(self) -> str: return self.emitter.__repr__() diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index a6393aa9d0ced..3b77d58a8711b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -1,7 +1,9 @@ +import datetime import json import logging from collections import defaultdict from dataclasses import dataclass, field as dataclass_field +from functools import lru_cache from typing import ( Any, DefaultDict, @@ -117,9 +119,10 @@ class GlueSourceConfig( description=f"The platform to use for the dataset URNs. Must be one of {VALID_PLATFORMS}.", ) + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-table-tableinput.html#cfn-glue-table-tableinput-owner extract_owners: Optional[bool] = Field( default=True, - description="When enabled, extracts ownership from Glue directly and overwrites existing owners. When disabled, ownership is left empty for datasets.", + description="When enabled, extracts ownership from Glue table property and overwrites existing owners (DATAOWNER). When disabled, ownership is left empty for datasets. Expects a corpGroup urn, a corpuser urn or only the identifier part for the latter. Not used in the normal course of AWS Glue operations.", ) extract_transforms: Optional[bool] = Field( default=True, description="Whether to extract Glue transform jobs." @@ -895,6 +898,12 @@ def gen_database_containers( ) -> Iterable[MetadataWorkUnit]: domain_urn = self._gen_domain_urn(database["Name"]) database_container_key = self.gen_database_key(database["Name"]) + parameters = database.get("Parameters", {}) + if database.get("LocationUri") is not None: + parameters["LocationUri"] = database["LocationUri"] + if database.get("CreateTime") is not None: + create_time: datetime.datetime = database["CreateTime"] + parameters["CreateTime"] = create_time.strftime("%B %d, %Y at %H:%M:%S") yield from gen_containers( container_key=database_container_key, name=database["Name"], @@ -904,6 +913,7 @@ def gen_database_containers( qualified_name=self.get_glue_arn( account_id=database["CatalogId"], database=database["Name"] ), + extra_properties=parameters, ) def add_table_to_database_container( @@ -941,9 +951,11 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: ] def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: - database_seen = set() databases, tables = self.get_all_databases_and_tables() + for database in databases.values(): + yield from self.gen_database_containers(database) + for table in tables: database_name = table["DatabaseName"] table_name = table["Name"] @@ -954,9 +966,6 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: ) or not self.source_config.table_pattern.allowed(full_table_name): self.report.report_table_dropped(full_table_name) continue - if database_name not in database_seen: - database_seen.add(database_name) - yield from self.gen_database_containers(databases[database_name]) dataset_urn = make_dataset_urn_with_platform_instance( platform=self.platform, @@ -1054,20 +1063,6 @@ def _extract_record( f"extract record from table={table_name} for dataset={dataset_urn}" ) - def get_owner() -> Optional[OwnershipClass]: - owner = table.get("Owner") - if owner: - owners = [ - OwnerClass( - owner=f"urn:li:corpuser:{owner}", - type=OwnershipTypeClass.DATAOWNER, - ) - ] - return OwnershipClass( - owners=owners, - ) - return None - def get_dataset_properties() -> DatasetPropertiesClass: return DatasetPropertiesClass( description=table.get("Description"), @@ -1277,6 +1272,20 @@ def get_data_platform_instance() -> DataPlatformInstanceClass: else None, ) + @lru_cache(maxsize=None) + def _get_ownership(owner: str) -> Optional[OwnershipClass]: + if owner: + owners = [ + OwnerClass( + owner=mce_builder.make_user_urn(owner), + type=OwnershipTypeClass.DATAOWNER, + ) + ] + return OwnershipClass( + owners=owners, + ) + return None + dataset_snapshot = DatasetSnapshot( urn=dataset_urn, aspects=[ @@ -1291,8 +1300,10 @@ def get_data_platform_instance() -> DataPlatformInstanceClass: dataset_snapshot.aspects.append(get_data_platform_instance()) + # Ownership if self.extract_owners: - optional_owner_aspect = get_owner() + owner = table.get("Owner") + optional_owner_aspect = _get_ownership(owner) if optional_owner_aspect is not None: dataset_snapshot.aspects.append(optional_owner_aspect) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py index eecc0f4372969..5046f52cdce26 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py @@ -130,6 +130,7 @@ ) from datahub.utilities.mapping import Constants from datahub.utilities.perf_timer import PerfTimer +from datahub.utilities.ratelimiter import RateLimiter from datahub.utilities.registries.domain_registry import DomainRegistry logger: logging.Logger = logging.getLogger(__name__) @@ -236,8 +237,14 @@ def __init__(self, ctx: PipelineContext, config: BigQueryV2Config): BigqueryTableIdentifier._BQ_SHARDED_TABLE_SUFFIX = "" self.bigquery_data_dictionary = BigQuerySchemaApi( - self.report.schema_api_perf, self.config.get_bigquery_client() + self.report.schema_api_perf, + self.config.get_bigquery_client(), ) + if self.config.extract_policy_tags_from_catalog: + self.bigquery_data_dictionary.datacatalog_client = ( + self.config.get_policy_tag_manager_client() + ) + self.sql_parser_schema_resolver = self._init_schema_resolver() self.data_reader: Optional[BigQueryDataReader] = None @@ -358,7 +365,7 @@ def metadata_read_capability_test( project_id=project_id, dataset_name=result[0].name, tables={}, - with_data_read_permission=config.is_profiling_enabled(), + with_data_read_permission=config.have_table_data_read_permission, ) if len(list(tables)) == 0: return CapabilityReport( @@ -637,7 +644,7 @@ def _query_project_list(self) -> Iterable[BigqueryProject]: "Maybe resourcemanager.projects.get permission is missing for the service account. " "You can assign predefined roles/bigquery.metadataViewer role to your service account.", ) - return [] + return for project in projects: if self.config.project_id_pattern.allowed(project.id): @@ -742,6 +749,12 @@ def _process_schema( columns = None + rate_limiter: Optional[RateLimiter] = None + if self.config.rate_limit: + rate_limiter = RateLimiter( + max_calls=self.config.requests_per_min, period=60 + ) + if ( self.config.include_tables or self.config.include_views @@ -752,6 +765,9 @@ def _process_schema( dataset_name=dataset_name, column_limit=self.config.column_limit, run_optimized_column_query=self.config.run_optimized_column_query, + extract_policy_tags_from_catalog=self.config.extract_policy_tags_from_catalog, + report=self.report, + rate_limiter=rate_limiter, ) if self.config.include_tables: @@ -1048,11 +1064,19 @@ def gen_view_dataset_workunits( project_id: str, dataset_name: str, ) -> Iterable[MetadataWorkUnit]: + tags_to_add = None + if table.labels and self.config.capture_view_label_as_tag: + tags_to_add = [ + make_tag_urn(f"{k}:{v}") + for k, v in table.labels.items() + if is_tag_allowed(self.config.capture_view_label_as_tag, k) + ] yield from self.gen_dataset_workunits( table=table, columns=columns, project_id=project_id, dataset_name=dataset_name, + tags_to_add=tags_to_add, sub_types=[DatasetSubTypes.VIEW], ) @@ -1275,6 +1299,9 @@ def gen_schema_fields(self, columns: List[BigqueryColumn]) -> List[SchemaField]: ) ) + if col.policy_tags: + for policy_tag in col.policy_tags: + tags.append(TagAssociationClass(make_tag_urn(policy_tag))) field = SchemaField( fieldPath=col.name, type=SchemaFieldDataType( @@ -1353,7 +1380,7 @@ def get_tables_for_dataset( project_id, dataset_name, items_to_get, - with_data_read_permission=self.config.is_profiling_enabled(), + with_data_read_permission=self.config.have_table_data_read_permission, ) items_to_get.clear() @@ -1362,7 +1389,7 @@ def get_tables_for_dataset( project_id, dataset_name, items_to_get, - with_data_read_permission=self.config.is_profiling_enabled(), + with_data_read_permission=self.config.have_table_data_read_permission, ) self.report.metadata_extraction_sec[f"{project_id}.{dataset_name}"] = round( diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py index 0e7e98b0e5e8f..f8fcea7c57545 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py @@ -192,7 +192,7 @@ def from_string_name(cls, ref: str) -> "BigQueryTableRef": def from_urn(cls, urn: str) -> "BigQueryTableRef": """Raises: ValueError if urn is not a valid BigQuery table URN.""" dataset_urn = DatasetUrn.create_from_string(urn) - split = dataset_urn.get_dataset_name().rsplit(".", 3) + split = dataset_urn.name.rsplit(".", 3) if len(split) == 3: project, dataset, table = split else: diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py index d1a6ed84a28ac..578c9dddbd2e4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py @@ -3,7 +3,7 @@ from datetime import timedelta from typing import Any, Dict, List, Optional, Union -from google.cloud import bigquery +from google.cloud import bigquery, datacatalog_v1 from google.cloud.logging_v2.client import Client as GCPLoggingClient from pydantic import Field, PositiveInt, PrivateAttr, root_validator, validator @@ -70,6 +70,9 @@ def get_bigquery_client(self) -> bigquery.Client: client_options = self.extra_client_options return bigquery.Client(self.project_on_behalf, **client_options) + def get_policy_tag_manager_client(self) -> datacatalog_v1.PolicyTagManagerClient: + return datacatalog_v1.PolicyTagManagerClient() + def make_gcp_logging_client( self, project_id: Optional[str] = None ) -> GCPLoggingClient: @@ -124,6 +127,11 @@ class BigQueryV2Config( description="Capture BigQuery table labels as DataHub tag", ) + capture_view_label_as_tag: Union[bool, AllowDenyPattern] = Field( + default=False, + description="Capture BigQuery view labels as DataHub tag", + ) + capture_dataset_label_as_tag: Union[bool, AllowDenyPattern] = Field( default=False, description="Capture BigQuery dataset labels as DataHub tag", @@ -176,6 +184,15 @@ class BigQueryV2Config( description="Number of partitioned table queried in batch when getting metadata. This is a low level config property which should be touched with care. This restriction is needed because we query partitions system view which throws error if we try to touch too many tables.", ) + use_tables_list_query_v2: bool = Field( + default=False, + description="List tables using an improved query that extracts partitions and last modified timestamps more accurately. Requires the ability to read table data. Automatically enabled when profiling is enabled.", + ) + + @property + def have_table_data_read_permission(self) -> bool: + return self.use_tables_list_query_v2 or self.is_profiling_enabled() + column_limit: int = Field( default=300, description="Maximum number of columns to process in a table. This is a low level config property which should be touched with care. This restriction is needed because excessively wide tables can result in failure to ingest the schema.", @@ -226,6 +243,16 @@ class BigQueryV2Config( description="Use the legacy sharded table urn suffix added.", ) + extract_policy_tags_from_catalog: bool = Field( + default=False, + description=( + "This flag enables the extraction of policy tags from the Google Data Catalog API. " + "When enabled, the extractor will fetch policy tags associated with BigQuery table columns. " + "For more information about policy tags and column-level security, refer to the documentation: " + "https://cloud.google.com/bigquery/docs/column-level-security-intro" + ), + ) + scheme: str = "bigquery" log_page_size: PositiveInt = Field( diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_helper.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_helper.py index 6142c96a5faa1..bbdf32da13621 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_helper.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_helper.py @@ -1,4 +1,5 @@ -from typing import Optional +import re +from typing import Dict, Optional def unquote_and_decode_unicode_escape_seq( @@ -17,3 +18,10 @@ def unquote_and_decode_unicode_escape_seq( cleaned_string = string.encode().decode("unicode-escape") return cleaned_string + + +def parse_labels(labels_str: str) -> Dict[str, str]: + pattern = r'STRUCT\("([^"]+)", "([^"]+)"\)' + + # Map of BigQuery label keys to label values + return dict(re.findall(pattern, labels_str)) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py index 54eca61dfe1c9..8a1bf9e5f3d1d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py @@ -20,6 +20,8 @@ @dataclass class BigQuerySchemaApiPerfReport(Report): + num_list_projects: int = 0 + num_list_projects_retry_request: int = 0 list_projects: PerfTimer = field(default_factory=PerfTimer) list_datasets: PerfTimer = field(default_factory=PerfTimer) get_columns_for_dataset: PerfTimer = field(default_factory=PerfTimer) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py index ca09496eda341..fe9bbc134a147 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py @@ -2,9 +2,10 @@ from collections import defaultdict from dataclasses import dataclass, field from datetime import datetime, timezone -from typing import Any, Dict, Iterator, List, Optional +from typing import Any, Dict, Iterable, Iterator, List, Optional -from google.cloud import bigquery +from google.api_core import retry +from google.cloud import bigquery, datacatalog_v1 from google.cloud.bigquery.table import ( RowIterator, TableListItem, @@ -13,6 +14,7 @@ ) from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier +from datahub.ingestion.source.bigquery_v2.bigquery_helper import parse_labels from datahub.ingestion.source.bigquery_v2.bigquery_report import ( BigQuerySchemaApiPerfReport, BigQueryV2Report, @@ -22,6 +24,7 @@ BigqueryTableType, ) from datahub.ingestion.source.sql.sql_generic import BaseColumn, BaseTable, BaseView +from datahub.utilities.ratelimiter import RateLimiter logger: logging.Logger = logging.getLogger(__name__) @@ -31,6 +34,7 @@ class BigqueryColumn(BaseColumn): field_path: str is_partition_column: bool cluster_column_position: Optional[int] + policy_tags: Optional[List[str]] = None RANGE_PARTITION_NAME: str = "RANGE" @@ -51,9 +55,7 @@ def from_time_partitioning( cls, time_partitioning: TimePartitioning ) -> "PartitionInfo": return cls( - field=time_partitioning.field - if time_partitioning.field - else "_PARTITIONTIME", + field=time_partitioning.field or "_PARTITIONTIME", type=time_partitioning.type_, expiration_ms=time_partitioning.expiration_ms, require_partition_filter=time_partitioning.require_partition_filter, @@ -104,6 +106,7 @@ class BigqueryTable(BaseTable): class BigqueryView(BaseView): columns: List[BigqueryColumn] = field(default_factory=list) materialized: bool = False + labels: Optional[Dict[str, str]] = None @dataclass @@ -137,10 +140,14 @@ class BigqueryProject: class BigQuerySchemaApi: def __init__( - self, report: BigQuerySchemaApiPerfReport, client: bigquery.Client + self, + report: BigQuerySchemaApiPerfReport, + client: bigquery.Client, + datacatalog_client: Optional[datacatalog_v1.PolicyTagManagerClient] = None, ) -> None: self.bq_client = client self.report = report + self.datacatalog_client = datacatalog_client def get_query_result(self, query: str) -> RowIterator: logger.debug(f"Query : {query}") @@ -148,14 +155,33 @@ def get_query_result(self, query: str) -> RowIterator: return resp.result() def get_projects(self) -> List[BigqueryProject]: + def _should_retry(exc: BaseException) -> bool: + logger.debug( + f"Exception occured for project.list api. Reason: {exc}. Retrying api request..." + ) + self.report.num_list_projects_retry_request += 1 + return True + with self.report.list_projects: try: - projects = self.bq_client.list_projects() - - return [ + # Bigquery API has limit in calling project.list request i.e. 2 request per second. + # https://cloud.google.com/bigquery/quotas#api_request_quotas + # Whenever this limit reached an exception occur with msg + # 'Quota exceeded: Your user exceeded quota for concurrent project.lists requests.' + # Hence, added the api request retry of 15 min. + # We already tried adding rate_limit externally, proving max_result and page_size + # to restrict the request calls inside list_project but issue still occured. + projects_iterator = self.bq_client.list_projects( + retry=retry.Retry( + predicate=_should_retry, initial=10, maximum=180, timeout=900 + ) + ) + projects: List[BigqueryProject] = [ BigqueryProject(id=p.project_id, name=p.friendly_name) - for p in projects + for p in projects_iterator ] + self.report.num_list_projects = len(projects) + return projects except Exception as e: logger.error(f"Error getting projects. {e}", exc_info=True) return [] @@ -219,9 +245,11 @@ def get_tables_for_dataset( BigqueryQuery.tables_for_dataset.format( project_id=project_id, dataset_name=dataset_name, - table_filter=f" and t.table_name in ({filter_clause})" - if filter_clause - else "", + table_filter=( + f" and t.table_name in ({filter_clause})" + if filter_clause + else "" + ), ), ) else: @@ -231,9 +259,11 @@ def get_tables_for_dataset( BigqueryQuery.tables_for_dataset_without_partition_data.format( project_id=project_id, dataset_name=dataset_name, - table_filter=f" and t.table_name in ({filter_clause})" - if filter_clause - else "", + table_filter=( + f" and t.table_name in ({filter_clause})" + if filter_clause + else "" + ), ), ) @@ -271,20 +301,22 @@ def _make_bigquery_table( return BigqueryTable( name=table.table_name, created=table.created, - last_altered=datetime.fromtimestamp( - table.get("last_altered") / 1000, tz=timezone.utc - ) - if table.get("last_altered") is not None - else None, + last_altered=( + datetime.fromtimestamp( + table.get("last_altered") / 1000, tz=timezone.utc + ) + if table.get("last_altered") is not None + else None + ), size_in_bytes=table.get("bytes"), rows_count=table.get("row_count"), comment=table.comment, ddl=table.ddl, expires=expiration, labels=table_basic.labels if table_basic else None, - partition_info=PartitionInfo.from_table_info(table_basic) - if table_basic - else None, + partition_info=( + PartitionInfo.from_table_info(table_basic) if table_basic else None + ), clustering_fields=table_basic.clustering_fields if table_basic else None, max_partition_id=table.get("max_partition_id"), max_shard_id=shard, @@ -335,37 +367,97 @@ def _make_bigquery_view(view: bigquery.Row) -> BigqueryView: return BigqueryView( name=view.table_name, created=view.created, - last_altered=datetime.fromtimestamp( - view.get("last_altered") / 1000, tz=timezone.utc - ) - if view.get("last_altered") is not None - else None, + last_altered=( + datetime.fromtimestamp(view.get("last_altered") / 1000, tz=timezone.utc) + if view.get("last_altered") is not None + else None + ), comment=view.comment, view_definition=view.view_definition, materialized=view.table_type == BigqueryTableType.MATERIALIZED_VIEW, size_in_bytes=view.get("size_bytes"), rows_count=view.get("row_count"), + labels=parse_labels(view.labels) if view.get("labels") else None, ) + def get_policy_tags_for_column( + self, + project_id: str, + dataset_name: str, + table_name: str, + column_name: str, + report: BigQueryV2Report, + rate_limiter: Optional[RateLimiter] = None, + ) -> Iterable[str]: + assert self.datacatalog_client + + try: + # Get the table schema + table_ref = f"{project_id}.{dataset_name}.{table_name}" + table = self.bq_client.get_table(table_ref) + schema = table.schema + + # Find the specific field in the schema + field = next((f for f in schema if f.name == column_name), None) + if not field or not field.policy_tags: + return + + # Retrieve policy tag display names + for policy_tag_name in field.policy_tags.names: + try: + if rate_limiter: + with rate_limiter: + policy_tag = self.datacatalog_client.get_policy_tag( + name=policy_tag_name + ) + else: + policy_tag = self.datacatalog_client.get_policy_tag( + name=policy_tag_name + ) + yield policy_tag.display_name + except Exception as e: + logger.warning( + f"Unexpected error when retrieving policy tag {policy_tag_name} for column {column_name} in table {table_name}: {e}", + exc_info=True, + ) + report.report_warning( + "metadata-extraction", + f"Failed to retrieve policy tag {policy_tag_name} for column {column_name} in table {table_name} due to unexpected error: {e}", + ) + except Exception as e: + logger.error( + f"Unexpected error retrieving schema for table {table_name} in dataset {dataset_name}, project {project_id}: {e}", + exc_info=True, + ) + report.report_warning( + "metadata-extraction", + f"Failed to retrieve schema for table {table_name} in dataset {dataset_name}, project {project_id} due to unexpected error: {e}", + ) + def get_columns_for_dataset( self, project_id: str, dataset_name: str, column_limit: int, + report: BigQueryV2Report, run_optimized_column_query: bool = False, + extract_policy_tags_from_catalog: bool = False, + rate_limiter: Optional[RateLimiter] = None, ) -> Optional[Dict[str, List[BigqueryColumn]]]: columns: Dict[str, List[BigqueryColumn]] = defaultdict(list) with self.report.get_columns_for_dataset: try: cur = self.get_query_result( - BigqueryQuery.columns_for_dataset.format( - project_id=project_id, dataset_name=dataset_name - ) - if not run_optimized_column_query - else BigqueryQuery.optimized_columns_for_dataset.format( - project_id=project_id, - dataset_name=dataset_name, - column_limit=column_limit, + ( + BigqueryQuery.columns_for_dataset.format( + project_id=project_id, dataset_name=dataset_name + ) + if not run_optimized_column_query + else BigqueryQuery.optimized_columns_for_dataset.format( + project_id=project_id, + dataset_name=dataset_name, + column_limit=column_limit, + ) ), ) except Exception as e: @@ -397,6 +489,20 @@ def get_columns_for_dataset( comment=column.comment, is_partition_column=column.is_partitioning_column == "YES", cluster_column_position=column.clustering_ordinal_position, + policy_tags=( + list( + self.get_policy_tags_for_column( + project_id, + dataset_name, + column.table_name, + column.column_name, + report, + rate_limiter, + ) + ) + if extract_policy_tags_from_catalog + else [] + ), ) ) @@ -483,11 +589,13 @@ def _make_bigquery_table_snapshot(snapshot: bigquery.Row) -> BigqueryTableSnapsh return BigqueryTableSnapshot( name=snapshot.table_name, created=snapshot.created, - last_altered=datetime.fromtimestamp( - snapshot.get("last_altered") / 1000, tz=timezone.utc - ) - if snapshot.get("last_altered") is not None - else None, + last_altered=( + datetime.fromtimestamp( + snapshot.get("last_altered") / 1000, tz=timezone.utc + ) + if snapshot.get("last_altered") is not None + else None + ), comment=snapshot.comment, ddl=snapshot.ddl, snapshot_time=snapshot.snapshot_time, diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py index 3545cc7743838..6155619d144c4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py @@ -117,7 +117,8 @@ class BigqueryQuery: t.table_type as table_type, t.creation_time as created, ts.last_modified_time as last_altered, - tos.OPTION_VALUE as comment, + tos_description.OPTION_VALUE as comment, + tos_labels.OPTION_VALUE as labels, t.is_insertable_into, t.ddl as view_definition, ts.row_count, @@ -125,9 +126,12 @@ class BigqueryQuery: FROM `{{project_id}}`.`{{dataset_name}}`.INFORMATION_SCHEMA.TABLES t join `{{project_id}}`.`{{dataset_name}}`.__TABLES__ as ts on ts.table_id = t.TABLE_NAME - left join `{{project_id}}`.`{{dataset_name}}`.INFORMATION_SCHEMA.TABLE_OPTIONS as tos on t.table_schema = tos.table_schema - and t.TABLE_NAME = tos.TABLE_NAME - and tos.OPTION_NAME = "description" + left join `{{project_id}}`.`{{dataset_name}}`.INFORMATION_SCHEMA.TABLE_OPTIONS as tos_description on t.table_schema = tos_description.table_schema + and t.TABLE_NAME = tos_description.TABLE_NAME + and tos_description.OPTION_NAME = "description" + left join `{{project_id}}`.`{{dataset_name}}`.INFORMATION_SCHEMA.TABLE_OPTIONS as tos_labels on t.table_schema = tos_labels.table_schema + and t.TABLE_NAME = tos_labels.TABLE_NAME + and tos_labels.OPTION_NAME = "labels" WHERE table_type in ('{BigqueryTableType.VIEW}', '{BigqueryTableType.MATERIALIZED_VIEW}') order by @@ -142,14 +146,18 @@ class BigqueryQuery: t.table_name as table_name, t.table_type as table_type, t.creation_time as created, - tos.OPTION_VALUE as comment, + tos_description.OPTION_VALUE as comment, + tos_labels.OPTION_VALUE as labels, t.is_insertable_into, t.ddl as view_definition FROM `{{project_id}}`.`{{dataset_name}}`.INFORMATION_SCHEMA.TABLES t - left join `{{project_id}}`.`{{dataset_name}}`.INFORMATION_SCHEMA.TABLE_OPTIONS as tos on t.table_schema = tos.table_schema - and t.TABLE_NAME = tos.TABLE_NAME - and tos.OPTION_NAME = "description" + left join `{{project_id}}`.`{{dataset_name}}`.INFORMATION_SCHEMA.TABLE_OPTIONS as tos_description on t.table_schema = tos_description.table_schema + and t.TABLE_NAME = tos_description.TABLE_NAME + and tos_description.OPTION_NAME = "description" + left join `{{project_id}}`.`{{dataset_name}}`.INFORMATION_SCHEMA.TABLE_OPTIONS as tos_labels on t.table_schema = tos_labels.table_schema + and t.TABLE_NAME = tos_labels.TABLE_NAME + and tos_labels.OPTION_NAME = "labels" WHERE table_type in ('{BigqueryTableType.VIEW}', '{BigqueryTableType.MATERIALIZED_VIEW}') order by diff --git a/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py b/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py index 54475cb509621..fba71240282c4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py +++ b/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py @@ -231,16 +231,25 @@ def get_schemas_from_confluent_ref_json( return all_schemas def _get_schema_and_fields( - self, topic: str, is_key_schema: bool + self, topic: str, is_key_schema: bool, is_subject: bool ) -> Tuple[Optional[Schema], List[SchemaField]]: schema: Optional[Schema] = None - schema_type_str: str = "key" if is_key_schema else "value" - topic_subject: Optional[str] = self._get_subject_for_topic( - topic=topic, is_key_schema=is_key_schema - ) + kafka_entity = "subject" if is_subject else "topic" + + # if provided schema as topic, assuming it as value subject + schema_type_str: Optional[str] = "value" + topic_subject: Optional[str] = None + if not is_subject: + schema_type_str = "key" if is_key_schema else "value" + topic_subject = self._get_subject_for_topic( + topic=topic, is_key_schema=is_key_schema + ) + else: + topic_subject = topic + if topic_subject is not None: logger.debug( - f"The {schema_type_str} schema subject:'{topic_subject}' is found for topic:'{topic}'." + f"The {schema_type_str} schema subject:'{topic_subject}' is found for {kafka_entity}:'{topic}'." ) try: registered_schema = self.schema_registry_client.get_latest_version( @@ -249,7 +258,7 @@ def _get_schema_and_fields( schema = registered_schema.schema except Exception as e: logger.warning( - f"For topic: {topic}, failed to get {schema_type_str} schema from schema registry using subject:'{topic_subject}': {e}." + f"For {kafka_entity}: {topic}, failed to get {schema_type_str} schema from schema registry using subject:'{topic_subject}': {e}." ) self.report.report_warning( topic, @@ -257,21 +266,23 @@ def _get_schema_and_fields( ) else: logger.debug( - f"For topic: {topic}, the schema registry subject for the {schema_type_str} schema is not found." + f"For {kafka_entity}: {topic}, the schema registry subject for the {schema_type_str} schema is not found." ) if not is_key_schema: # Value schema is always expected. Report a warning. self.report.report_warning( topic, f"The schema registry subject for the {schema_type_str} schema is not found." - f" The topic is either schema-less, or no messages have been written to the topic yet.", + f" The {kafka_entity} is either schema-less, or no messages have been written to the {kafka_entity} yet.", ) # Obtain the schema fields from schema for the topic. fields: List[SchemaField] = [] if schema is not None: fields = self._get_schema_fields( - topic=topic, schema=schema, is_key_schema=is_key_schema + topic=topic, + schema=schema, + is_key_schema=is_key_schema, ) return (schema, fields) @@ -352,16 +363,21 @@ def _get_schema_fields( return fields def _get_schema_metadata( - self, topic: str, platform_urn: str + self, topic: str, platform_urn: str, is_subject: bool ) -> Optional[SchemaMetadata]: + # Process the value schema schema, fields = self._get_schema_and_fields( - topic=topic, is_key_schema=False + topic=topic, + is_key_schema=False, + is_subject=is_subject, ) # type: Tuple[Optional[Schema], List[SchemaField]] # Process the key schema key_schema, key_fields = self._get_schema_and_fields( - topic=topic, is_key_schema=True + topic=topic, + is_key_schema=True, + is_subject=is_subject, ) # type:Tuple[Optional[Schema], List[SchemaField]] # Create the schemaMetadata aspect. @@ -388,17 +404,22 @@ def _get_schema_metadata( return None def get_schema_metadata( - self, topic: str, platform_urn: str + self, topic: str, platform_urn: str, is_subject: bool ) -> Optional[SchemaMetadata]: - logger.debug(f"Inside _get_schema_metadata {topic} {platform_urn}") + logger.debug(f"Inside get_schema_metadata {topic} {platform_urn}") + # Process the value schema schema, fields = self._get_schema_and_fields( - topic=topic, is_key_schema=False + topic=topic, + is_key_schema=False, + is_subject=is_subject, ) # type: Tuple[Optional[Schema], List[SchemaField]] # Process the key schema key_schema, key_fields = self._get_schema_and_fields( - topic=topic, is_key_schema=True + topic=topic, + is_key_schema=True, + is_subject=is_subject, ) # type:Tuple[Optional[Schema], List[SchemaField]] # Create the schemaMetadata aspect. @@ -423,3 +444,6 @@ def get_schema_metadata( fields=key_fields + fields, ) return None + + def get_subjects(self) -> List[str]: + return self.known_schema_registry_subjects diff --git a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py index d998c37d32ed2..feee89ba57983 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py +++ b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py @@ -93,13 +93,18 @@ class CSVEnricherReport(SourceReport): num_domain_workunits_produced: int = 0 -@platform_name("CSV") +@platform_name("CSV Enricher") @config_class(CSVEnricherConfig) @support_status(SupportStatus.INCUBATING) class CSVEnricherSource(Source): """ + :::tip Looking to ingest a CSV data file into DataHub, as an asset? + Use the [Local File](./s3.md) ingestion source. + The CSV enricher is used for enriching entities already ingested into DataHub. + ::: + This plugin is used to bulk upload metadata to Datahub. - It will apply glossary terms, tags, decription, owners and domain at the entity level. It can also be used to apply tags, + It will apply glossary terms, tags, description, owners and domain at the entity level. It can also be used to apply tags, glossary terms, and documentation at the column level. These values are read from a CSV file. You have the option to either overwrite or append existing values. diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py index 820d85b2cfb51..8a99f096b5167 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py @@ -116,6 +116,7 @@ def set_metadata_endpoint(cls, values: dict) -> dict: """ _DBT_GRAPHQL_MODEL_SEED_SNAPSHOT_FIELDS = """ + packageName alias error status @@ -433,6 +434,7 @@ def _parse_into_dbt_node(self, node: Dict) -> DBTNode: dbt_name=key, # TODO: Get the dbt adapter natively. dbt_adapter=self.config.target_platform, + dbt_package_name=node.get("packageName"), database=node.get("database"), schema=node.get("schema"), name=name, diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py index b758f218e2586..f56784ab1f082 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py @@ -502,6 +502,7 @@ class DBTNode: dbt_adapter: str dbt_name: str dbt_file_path: Optional[str] + dbt_package_name: Optional[str] # this is pretty much always present node_type: str # source, model, snapshot, seed, test, etc max_loaded_at: Optional[datetime] @@ -644,6 +645,7 @@ def get_custom_properties(node: DBTNode) -> Dict[str, str]: "catalog_type": node.catalog_type, "language": node.language, "dbt_unique_id": node.dbt_name, + "dbt_package_name": node.dbt_package_name, } for attribute, node_attribute_value in node_attributes.items(): @@ -1249,9 +1251,6 @@ def create_dbt_platform_mces( ) -> Iterable[MetadataWorkUnit]: """Create MCEs and MCPs for the dbt platform.""" - mce_platform = DBT_PLATFORM - mce_platform_instance = self.config.platform_instance - action_processor = OperationProcessor( self.config.meta_mapping, self.config.tag_prefix, @@ -1267,15 +1266,10 @@ def create_dbt_platform_mces( ) for node in sorted(dbt_nodes, key=lambda n: n.dbt_name): node_datahub_urn = node.get_urn( - mce_platform, + DBT_PLATFORM, self.config.env, - mce_platform_instance, + self.config.platform_instance, ) - if not self.config.entities_enabled.can_emit_node_type(node.node_type): - logger.debug( - f"Skipping emission of node {node_datahub_urn} because node_type {node.node_type} is disabled" - ) - continue meta_aspects: Dict[str, Any] = {} if self.config.enable_meta_mapping and node.meta: @@ -1287,7 +1281,7 @@ def create_dbt_platform_mces( ) # mutates meta_aspects aspects = self._generate_base_dbt_aspects( - node, additional_custom_props_filtered, mce_platform, meta_aspects + node, additional_custom_props_filtered, DBT_PLATFORM, meta_aspects ) # Upstream lineage. @@ -1302,29 +1296,36 @@ def create_dbt_platform_mces( if view_prop_aspect: aspects.append(view_prop_aspect) - # Subtype. - sub_type_wu = self._create_subType_wu(node, node_datahub_urn) - if sub_type_wu: - yield sub_type_wu + # Generate main MCE. + if self.config.entities_enabled.can_emit_node_type(node.node_type): + # Subtype. + sub_type_wu = self._create_subType_wu(node, node_datahub_urn) + if sub_type_wu: + yield sub_type_wu - # DataPlatformInstance aspect. - yield MetadataChangeProposalWrapper( - entityUrn=node_datahub_urn, - aspect=self._make_data_platform_instance_aspect(), - ).as_workunit() + # DataPlatformInstance aspect. + yield MetadataChangeProposalWrapper( + entityUrn=node_datahub_urn, + aspect=self._make_data_platform_instance_aspect(), + ).as_workunit() - if len(aspects) == 0: - continue - dataset_snapshot = DatasetSnapshot(urn=node_datahub_urn, aspects=aspects) - mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) - if self.config.write_semantics == "PATCH": - mce = self.get_patched_mce(mce) - yield MetadataWorkUnit(id=dataset_snapshot.urn, mce=mce) + dataset_snapshot = DatasetSnapshot( + urn=node_datahub_urn, aspects=aspects + ) + mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) + if self.config.write_semantics == "PATCH": + mce = self.get_patched_mce(mce) + yield MetadataWorkUnit(id=dataset_snapshot.urn, mce=mce) + else: + logger.debug( + f"Skipping emission of node {node_datahub_urn} because node_type {node.node_type} is disabled" + ) # Model performance. - yield from auto_workunit( - self._create_dataprocess_instance_mcps(node, upstream_lineage_class) - ) + if self.config.entities_enabled.can_emit_model_performance: + yield from auto_workunit( + self._create_dataprocess_instance_mcps(node, upstream_lineage_class) + ) def _create_dataprocess_instance_mcps( self, @@ -1333,8 +1334,6 @@ def _create_dataprocess_instance_mcps( ) -> Iterable[MetadataChangeProposalWrapper]: if not node.model_performances: return - if not self.config.entities_enabled.can_emit_model_performance: - return node_datahub_urn = node.get_urn( DBT_PLATFORM, @@ -1484,7 +1483,7 @@ def get_patched_mce(self, mce): transformed_tag_list = self.get_transformed_tags_by_prefix( tag_aspect.tags, mce.proposedSnapshot.urn, - mce_builder.make_tag_urn(self.config.tag_prefix), + tag_prefix=self.config.tag_prefix, ) tag_aspect.tags = transformed_tag_list @@ -1875,16 +1874,19 @@ def get_transformed_tags_by_prefix( self, new_tags: List[TagAssociationClass], entity_urn: str, - tags_prefix_filter: str, + tag_prefix: str, ) -> List[TagAssociationClass]: tag_set = {new_tag.tag for new_tag in new_tags} if self.ctx.graph: existing_tags_class = self.ctx.graph.get_tags(entity_urn) if existing_tags_class and existing_tags_class.tags: - for exiting_tag in existing_tags_class.tags: - if not exiting_tag.tag.startswith(tags_prefix_filter): - tag_set.add(exiting_tag.tag) + for existing_tag in existing_tags_class.tags: + if tag_prefix and existing_tag.tag.startswith( + mce_builder.make_tag_urn(tag_prefix) + ): + continue + tag_set.add(existing_tag.tag) return [TagAssociationClass(tag) for tag in sorted(tag_set)] # This method attempts to read-modify and return the glossary terms of a dataset. diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py index c78cfdf0b4f0f..e24c18147e4e6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py @@ -245,6 +245,7 @@ def extract_dbt_entities( dbtNode = DBTNode( dbt_name=key, dbt_adapter=manifest_adapter, + dbt_package_name=manifest_node.get("package_name"), database=manifest_node["database"], schema=manifest_node["schema"], name=name, diff --git a/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py b/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py index e3933b985c28a..6cab0ffc8f25c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py @@ -466,8 +466,9 @@ def construct_schema_metadata( if schema_size > MAX_SCHEMA_SIZE: # downsample the schema, using frequency as the sort key self.report.report_warning( - key=dataset_urn, - reason=f"Downsampling the table schema because MAX_SCHEMA_SIZE threshold is {MAX_SCHEMA_SIZE}", + title="Schema Size Too Large", + message=f"Downsampling the table schema because MAX_SCHEMA_SIZE threshold is {MAX_SCHEMA_SIZE}", + context=f"Collection: {dataset_urn}", ) # Add this information to the custom properties so user can know they are looking at down sampled schema @@ -535,7 +536,9 @@ def get_native_type(self, attribute_type: Union[type, str], table_name: str) -> ) if type_string is None: self.report.report_warning( - table_name, f"unable to map type {attribute_type} to native data type" + title="Unable to Map Attribute Type", + message=f"Unable to map type {attribute_type} to native data type", + context=f"Collection: {table_name}", ) return _attribute_type_to_native_type_mapping[attribute_type] return type_string @@ -550,8 +553,9 @@ def get_field_type( if type_class is None: self.report.report_warning( - table_name, - f"unable to map type {attribute_type} to metadata schema field type", + title="Unable to Map Field Type", + message=f"Unable to map type {attribute_type} to metadata schema field type", + context=f"Collection: {table_name}", ) type_class = NullTypeClass return SchemaFieldDataType(type=type_class()) diff --git a/metadata-ingestion/src/datahub/ingestion/source/file.py b/metadata-ingestion/src/datahub/ingestion/source/file.py index 49cc314426eb5..853487b1f1c9f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/file.py +++ b/metadata-ingestion/src/datahub/ingestion/source/file.py @@ -6,16 +6,13 @@ from dataclasses import dataclass, field from enum import auto from functools import partial -from io import BufferedReader from typing import Any, Iterable, Iterator, List, Optional, Tuple, Union -from urllib import parse import ijson -import requests from pydantic import validator from pydantic.fields import Field -from datahub.configuration.common import ConfigEnum, ConfigModel, ConfigurationError +from datahub.configuration.common import ConfigEnum, ConfigModel from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated from datahub.configuration.validate_field_rename import pydantic_renamed_field from datahub.emitter.mcp import MetadataChangeProposalWrapper @@ -35,6 +32,8 @@ ) from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.fs.fs_base import FileInfo, get_path_schema +from datahub.ingestion.fs.fs_registry import fs_registry from datahub.metadata.com.linkedin.pegasus2avro.mxe import ( MetadataChangeEvent, MetadataChangeProposal, @@ -56,11 +55,17 @@ class FileSourceConfig(ConfigModel): message="filename is deprecated. Use path instead.", ) path: str = Field( - description="File path to folder or file to ingest, or URL to a remote file. If pointed to a folder, all files with extension {file_extension} (default json) within that folder will be processed." + description=( + "File path to folder or file to ingest, or URL to a remote file. " + "If pointed to a folder, all files with extension {file_extension} (default json) within that folder will be processed." + ) ) file_extension: str = Field( ".json", - description="When providing a folder to use to read files, set this field to control file extensions that you want the source to process. * is a special value that means process every file regardless of extension", + description=( + "When providing a folder to use to read files, set this field to control file extensions that you want the source to process. " + "* is a special value that means process every file regardless of extension" + ), ) read_mode: FileReadMode = FileReadMode.AUTO aspect: Optional[str] = Field( @@ -69,7 +74,10 @@ class FileSourceConfig(ConfigModel): ) count_all_before_starting: bool = Field( default=True, - description="When enabled, counts total number of records in the file before starting. Used for accurate estimation of completion time. Turn it off if startup time is too high.", + description=( + "When enabled, counts total number of records in the file before starting. " + "Used for accurate estimation of completion time. Turn it off if startup time is too high." + ), ) _minsize_for_streaming_mode_in_bytes: int = ( @@ -163,46 +171,36 @@ def compute_stats(self) -> None: self.percentage_completion = f"{percentage_completion:.2f}%" -@platform_name("File") +@platform_name("Metadata File") @config_class(FileSourceConfig) @support_status(SupportStatus.CERTIFIED) class GenericFileSource(TestableSource): """ - This plugin pulls metadata from a previously generated file. The [file sink](../../../../metadata-ingestion/sink_docs/file.md) can produce such files, and a number of samples are included in the [examples/mce_files](../../../../metadata-ingestion/examples/mce_files) directory. + This plugin pulls metadata from a previously generated file. + The [metadata file sink](../../../../metadata-ingestion/sink_docs/metadata-file.md) can produce such files, and a number of + samples are included in the [examples/mce_files](../../../../metadata-ingestion/examples/mce_files) directory. """ def __init__(self, ctx: PipelineContext, config: FileSourceConfig): self.ctx = ctx self.config = config self.report = FileSourceReport() - self.fp: Optional[BufferedReader] = None @classmethod def create(cls, config_dict, ctx): config = FileSourceConfig.parse_obj(config_dict) return cls(ctx, config) - def get_filenames(self) -> Iterable[str]: - path_parsed = parse.urlparse(str(self.config.path)) - if path_parsed.scheme in ("file", ""): - path = pathlib.Path(self.config.path) - if path.is_file(): - self.report.total_num_files = 1 - return [str(path)] - elif path.is_dir(): - files_and_stats = [ - (str(x), os.path.getsize(x)) - for x in path.glob(f"*{self.config.file_extension}") - if x.is_file() - ] - self.report.total_num_files = len(files_and_stats) - self.report.total_bytes_on_disk = sum([y for (x, y) in files_and_stats]) - return [x for (x, y) in files_and_stats] - else: - raise Exception(f"Failed to process {path}") - else: - self.report.total_num_files = 1 - return [str(self.config.path)] + def get_filenames(self) -> Iterable[FileInfo]: + path_str = str(self.config.path) + schema = get_path_schema(path_str) + fs_class = fs_registry.get(schema) + fs = fs_class.create() + for file_info in fs.list(path_str): + if file_info.is_file and file_info.path.endswith( + self.config.file_extension + ): + yield file_info def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: # No super() call, as we don't want helpers that create / remove workunits @@ -213,7 +211,7 @@ def get_workunits_internal( ) -> Iterable[MetadataWorkUnit]: for f in self.get_filenames(): for i, obj in self.iterate_generic_file(f): - id = f"file://{f}:{i}" + id = f"{f.path}:{i}" if isinstance( obj, (MetadataChangeProposalWrapper, MetadataChangeProposal) ): @@ -230,99 +228,88 @@ def get_workunits_internal( yield MetadataWorkUnit(id, mcp_raw=obj) else: yield MetadataWorkUnit(id, mce=obj) + self.report.total_num_files += 1 + self.report.append_total_bytes_on_disk(f.size) def get_report(self): return self.report def close(self): - if self.fp: - self.fp.close() super().close() - def _iterate_file(self, path: str) -> Iterable[Tuple[int, Any]]: - self.report.current_file_name = path - path_parsed = parse.urlparse(path) - if path_parsed.scheme not in ("http", "https"): # A local file - self.report.current_file_size = os.path.getsize(path) - if self.config.read_mode == FileReadMode.AUTO: - file_read_mode = ( - FileReadMode.BATCH - if self.report.current_file_size - < self.config._minsize_for_streaming_mode_in_bytes - else FileReadMode.STREAM - ) - logger.info(f"Reading file {path} in {file_read_mode} mode") + def _iterate_file(self, file_status: FileInfo) -> Iterable[Any]: + file_read_mode = self.config.read_mode + if file_read_mode == FileReadMode.AUTO: + if file_status.size < self.config._minsize_for_streaming_mode_in_bytes: + file_read_mode = FileReadMode.BATCH else: - file_read_mode = self.config.read_mode - - if file_read_mode == FileReadMode.BATCH: - with open(path) as f: - parse_start_time = datetime.datetime.now() - obj_list = json.load(f) - parse_end_time = datetime.datetime.now() - self.report.add_parse_time(parse_end_time - parse_start_time) - if not isinstance(obj_list, list): - obj_list = [obj_list] - count_start_time = datetime.datetime.now() - self.report.current_file_num_elements = len(obj_list) - self.report.add_count_time(datetime.datetime.now() - count_start_time) - self.report.current_file_elements_read = 0 - for i, obj in enumerate(obj_list): - yield i, obj - self.report.current_file_elements_read += 1 + file_read_mode = FileReadMode.STREAM + + # Open the file. + schema = get_path_schema(file_status.path) + fs_class = fs_registry.get(schema) + fs = fs_class.create() + self.report.current_file_name = file_status.path + self.report.current_file_size = file_status.size + fp = fs.open(file_status.path) + + with fp: + if file_read_mode == FileReadMode.STREAM: + yield from self._iterate_file_streaming(fp) else: - self.fp = open(path, "rb") - if self.config.count_all_before_starting: - count_start_time = datetime.datetime.now() - parse_stream = ijson.parse(self.fp, use_float=True) - total_elements = 0 - for row in ijson.items(parse_stream, "item", use_float=True): - total_elements += 1 - count_end_time = datetime.datetime.now() - self.report.add_count_time(count_end_time - count_start_time) - self.report.current_file_num_elements = total_elements - self.report.current_file_elements_read = 0 - self.fp.seek(0) - parse_start_time = datetime.datetime.now() - parse_stream = ijson.parse(self.fp, use_float=True) - rows_yielded = 0 - for row in ijson.items(parse_stream, "item", use_float=True): - parse_end_time = datetime.datetime.now() - self.report.add_parse_time(parse_end_time - parse_start_time) - rows_yielded += 1 - self.report.current_file_elements_read += 1 - yield rows_yielded, row - parse_start_time = datetime.datetime.now() - else: - try: - response = requests.get(path) - parse_start_time = datetime.datetime.now() - data = response.json() - except Exception as e: - raise ConfigurationError(f"Cannot read remote file {path}, error:{e}") - if not isinstance(data, list): - data = [data] - parse_end_time = datetime.datetime.now() - self.report.add_parse_time(parse_end_time - parse_start_time) - self.report.current_file_size = len(response.content) - self.report.current_file_elements_read = 0 - for i, obj in enumerate(data): - yield i, obj - self.report.current_file_elements_read += 1 + yield from self._iterate_file_batch(fp) - self.report.files_completed.append(path) + self.report.files_completed.append(file_status.path) self.report.num_files_completed += 1 self.report.total_bytes_read_completed_files += self.report.current_file_size self.report.reset_current_file_stats() + def _iterate_file_streaming(self, fp: Any) -> Iterable[Any]: + # Count the number of elements in the file. + if self.config.count_all_before_starting: + count_start_time = datetime.datetime.now() + parse_stream = ijson.parse(fp, use_float=True) + total_elements = 0 + for _row in ijson.items(parse_stream, "item", use_float=True): + total_elements += 1 + count_end_time = datetime.datetime.now() + self.report.add_count_time(count_end_time - count_start_time) + self.report.current_file_num_elements = total_elements + fp.seek(0) + + # Read the file. + self.report.current_file_elements_read = 0 + parse_start_time = datetime.datetime.now() + parse_stream = ijson.parse(fp, use_float=True) + for row in ijson.items(parse_stream, "item", use_float=True): + parse_end_time = datetime.datetime.now() + self.report.add_parse_time(parse_end_time - parse_start_time) + self.report.current_file_elements_read += 1 + yield row + + def _iterate_file_batch(self, fp: Any) -> Iterable[Any]: + # Read the file. + contents = json.load(fp) + + # Maintain backwards compatibility with the single-object format. + if isinstance(contents, list): + for row in contents: + yield row + else: + yield contents + def iterate_mce_file(self, path: str) -> Iterator[MetadataChangeEvent]: - for i, obj in self._iterate_file(path): + # TODO: Remove this method, as it appears to be unused. + schema = get_path_schema(path) + fs_class = fs_registry.get(schema) + fs = fs_class.create() + file_status = fs.file_status(path) + for obj in self._iterate_file(file_status): mce: MetadataChangeEvent = MetadataChangeEvent.from_obj(obj) yield mce def iterate_generic_file( - self, - path: str, + self, file_status: FileInfo ) -> Iterator[ Tuple[ int, @@ -333,7 +320,7 @@ def iterate_generic_file( ], ] ]: - for i, obj in self._iterate_file(path): + for i, obj in enumerate(self._iterate_file(file_status)): try: deserialize_start_time = datetime.datetime.now() item = _from_obj_for_file(obj) @@ -378,6 +365,11 @@ def test_connection(config_dict: dict) -> TestConnectionReport: basic_connectivity=CapabilityReport(capable=True) ) + @staticmethod + def close_if_possible(stream): + if hasattr(stream, "close") and callable(stream.close): + stream.close() + def _from_obj_for_file( obj: dict, diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py index a689e9ee642ae..f8b1c6dd93d6d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py @@ -9,9 +9,13 @@ from datahub.configuration.common import AllowDenyPattern, ConfigModel from datahub.configuration.source_common import DEFAULT_ENV, DatasetSourceConfigMixin from datahub.configuration.validate_field_rename import pydantic_renamed_field +from datahub.ingestion.api.report import Report from datahub.ingestion.source.bigquery_v2.bigquery_config import ( BigQueryConnectionConfig, ) +from datahub.ingestion.source.snowflake.snowflake_connection import ( + SnowflakeConnectionConfig, +) from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalSourceReport, StatefulStaleMetadataRemovalConfig, @@ -19,7 +23,7 @@ from datahub.ingestion.source.state.stateful_ingestion_base import ( StatefulIngestionConfigBase, ) -from datahub.ingestion.source_config.sql.snowflake import BaseSnowflakeConfig +from datahub.utilities.perf_timer import PerfTimer logger = logging.getLogger(__name__) @@ -64,7 +68,7 @@ class Constant: } -class SnowflakeDestinationConfig(BaseSnowflakeConfig): +class SnowflakeDestinationConfig(SnowflakeConnectionConfig): database: str = Field(description="The fivetran connector log database.") log_schema: str = Field(description="The fivetran connector log schema.") @@ -110,10 +114,26 @@ def validate_destination_platfrom_and_config(cls, values: Dict) -> Dict: return values +@dataclass +class MetadataExtractionPerfReport(Report): + connectors_metadata_extraction_sec: PerfTimer = dataclass_field( + default_factory=PerfTimer + ) + connectors_lineage_extraction_sec: PerfTimer = dataclass_field( + default_factory=PerfTimer + ) + connectors_jobs_extraction_sec: PerfTimer = dataclass_field( + default_factory=PerfTimer + ) + + @dataclass class FivetranSourceReport(StaleEntityRemovalSourceReport): connectors_scanned: int = 0 filtered_connectors: List[str] = dataclass_field(default_factory=list) + metadata_extraction_perf: MetadataExtractionPerfReport = dataclass_field( + default_factory=MetadataExtractionPerfReport + ) def report_connectors_scanned(self, count: int = 1) -> None: self.connectors_scanned += count @@ -163,3 +183,7 @@ class FivetranSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin default={}, description="A mapping of destination dataset to platform instance. Use destination id as key.", ) + history_sync_lookback_period: int = pydantic.Field( + 7, + description="The number of days to look back when extracting connectors' sync history.", + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/data_classes.py index 4ae71b990e5cd..18de2b01edd3b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/data_classes.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List, Optional +from typing import List @dataclass @@ -23,7 +23,7 @@ class Connector: paused: bool sync_frequency: int destination_id: str - user_email: Optional[str] + user_id: str table_lineage: List[TableLineage] jobs: List["Job"] diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py index c8ae779b602b8..c899fe04d2c48 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py +++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py @@ -173,11 +173,12 @@ def _generate_datajob_from_connector(self, connector: Connector) -> DataJob: env=self.config.env, platform_instance=self.config.platform_instance, ) + owner_email = self.audit_log.get_user_email(connector.user_id) datajob = DataJob( id=connector.connector_id, flow_urn=dataflow_urn, name=connector.connector_name, - owners={connector.user_email} if connector.user_email else set(), + owners={owner_email} if owner_email else set(), ) job_property_bag: Dict[str, str] = {} @@ -220,7 +221,7 @@ def _get_dpi_workunits( f"Status should be either SUCCESSFUL, FAILURE_WITH_TASK or CANCELED and it was " f"{job.status}" ) - return [] + return result = status_result_map[job.status] start_timestamp_millis = job.start_time * 1000 for mcp in dpi.generate_mcp( @@ -281,7 +282,9 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: """ logger.info("Fivetran plugin execution is started") connectors = self.audit_log.get_allowed_connectors_list( - self.config.connector_patterns, self.report + self.config.connector_patterns, + self.report, + self.config.history_sync_lookback_period, ) for connector in connectors: logger.info(f"Processing connector id: {connector.connector_id}") diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py index a9eb59f929799..d8ce68e8345ec 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_log_api.py @@ -1,7 +1,9 @@ +import functools import json import logging from typing import Any, Dict, List, Optional, Tuple +import sqlglot from sqlalchemy import create_engine from datahub.configuration.common import AllowDenyPattern, ConfigurationError @@ -76,6 +78,11 @@ def _initialize_fivetran_variables( ) def _query(self, query: str) -> List[Dict]: + # Automatically transpile snowflake query syntax to the target dialect. + if self.fivetran_log_config.destination_platform != "snowflake": + query = sqlglot.parse_one(query, dialect="snowflake").sql( + dialect=self.fivetran_log_config.destination_platform, pretty=True + ) logger.debug(f"Query : {query}") resp = self.engine.execute(query) return [row for row in resp] @@ -151,9 +158,14 @@ def _get_table_lineage( return table_lineage_list - def _get_all_connector_sync_logs(self) -> Dict[str, Dict]: + def _get_all_connector_sync_logs(self, syncs_interval: int) -> Dict[str, Dict]: sync_logs = {} - for row in self._query(self.fivetran_log_query.get_sync_logs_query()): + for row in self._query( + self.fivetran_log_query.get_sync_logs_query().format( + db_clause=self.fivetran_log_query.db_clause, + syncs_interval=syncs_interval, + ) + ): if row[Constant.CONNECTOR_ID] not in sync_logs: sync_logs[row[Constant.CONNECTOR_ID]] = { row[Constant.SYNC_ID]: { @@ -208,50 +220,62 @@ def _get_jobs_list( ) return jobs - def _get_user_email(self, user_id: Optional[str]) -> Optional[str]: + @functools.lru_cache() + def _get_users(self) -> Dict[str, str]: + users = self._query(self.fivetran_log_query.get_users_query()) + if not users: + return {} + return {user[Constant.USER_ID]: user[Constant.EMAIL] for user in users} + + def get_user_email(self, user_id: str) -> Optional[str]: if not user_id: return None - user_details = self._query( - self.fivetran_log_query.get_user_query(user_id=user_id) - ) + return self._get_users().get(user_id) - if not user_details: - return None + def _fill_connectors_table_lineage(self, connectors: List[Connector]) -> None: + table_lineage_metadata = self._get_connectors_table_lineage_metadata() + column_lineage_metadata = self._get_column_lineage_metadata() + for connector in connectors: + connector.table_lineage = self._get_table_lineage( + column_lineage_metadata=column_lineage_metadata, + table_lineage_result=table_lineage_metadata.get(connector.connector_id), + ) - return f"{user_details[0][Constant.EMAIL]}" + def _fill_connectors_jobs( + self, connectors: List[Connector], syncs_interval: int + ) -> None: + sync_logs = self._get_all_connector_sync_logs(syncs_interval) + for connector in connectors: + connector.jobs = self._get_jobs_list(sync_logs.get(connector.connector_id)) def get_allowed_connectors_list( - self, connector_patterns: AllowDenyPattern, report: FivetranSourceReport + self, + connector_patterns: AllowDenyPattern, + report: FivetranSourceReport, + syncs_interval: int, ) -> List[Connector]: connectors: List[Connector] = [] - sync_logs = self._get_all_connector_sync_logs() - table_lineage_metadata = self._get_connectors_table_lineage_metadata() - column_lineage_metadata = self._get_column_lineage_metadata() - connector_list = self._query(self.fivetran_log_query.get_connectors_query()) - for connector in connector_list: - if not connector_patterns.allowed(connector[Constant.CONNECTOR_NAME]): - report.report_connectors_dropped(connector[Constant.CONNECTOR_NAME]) - continue - connectors.append( - Connector( - connector_id=connector[Constant.CONNECTOR_ID], - connector_name=connector[Constant.CONNECTOR_NAME], - connector_type=connector[Constant.CONNECTOR_TYPE_ID], - paused=connector[Constant.PAUSED], - sync_frequency=connector[Constant.SYNC_FREQUENCY], - destination_id=connector[Constant.DESTINATION_ID], - user_email=self._get_user_email( - connector[Constant.CONNECTING_USER_ID] - ), - table_lineage=self._get_table_lineage( - column_lineage_metadata=column_lineage_metadata, - table_lineage_result=table_lineage_metadata.get( - connector[Constant.CONNECTOR_ID] - ), - ), - jobs=self._get_jobs_list( - sync_logs.get(connector[Constant.CONNECTOR_ID]) - ), + with report.metadata_extraction_perf.connectors_metadata_extraction_sec: + connector_list = self._query(self.fivetran_log_query.get_connectors_query()) + for connector in connector_list: + if not connector_patterns.allowed(connector[Constant.CONNECTOR_NAME]): + report.report_connectors_dropped(connector[Constant.CONNECTOR_NAME]) + continue + connectors.append( + Connector( + connector_id=connector[Constant.CONNECTOR_ID], + connector_name=connector[Constant.CONNECTOR_NAME], + connector_type=connector[Constant.CONNECTOR_TYPE_ID], + paused=connector[Constant.PAUSED], + sync_frequency=connector[Constant.SYNC_FREQUENCY], + destination_id=connector[Constant.DESTINATION_ID], + user_id=connector[Constant.CONNECTING_USER_ID], + table_lineage=[], + jobs=[], + ) ) - ) + with report.metadata_extraction_perf.connectors_lineage_extraction_sec: + self._fill_connectors_table_lineage(connectors) + with report.metadata_extraction_perf.connectors_jobs_extraction_sec: + self._fill_connectors_jobs(connectors, syncs_interval) return connectors diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py index 8f621bc3ffd06..d965f53ff554b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py @@ -1,4 +1,7 @@ class FivetranLogQuery: + # Note: All queries are written in Snowflake SQL. + # They will be transpiled to the target database's SQL dialect at runtime. + def __init__(self) -> None: # Select query db clause self.db_clause: str = "" @@ -10,35 +13,38 @@ def use_database(self, db_name: str) -> str: return f"use database {db_name}" def get_connectors_query(self) -> str: - return f""" - SELECT connector_id, - connecting_user_id, - connector_type_id, - connector_name, - paused, - sync_frequency, - destination_id - FROM {self.db_clause}connector - WHERE _fivetran_deleted = FALSE""" + return f"""\ +SELECT + connector_id, + connecting_user_id, + connector_type_id, + connector_name, + paused, + sync_frequency, + destination_id +FROM {self.db_clause}connector +WHERE + _fivetran_deleted = FALSE\ +""" - def get_user_query(self, user_id: str) -> str: + def get_users_query(self) -> str: return f""" SELECT id as user_id, given_name, family_name, email - FROM {self.db_clause}user - WHERE id = '{user_id}'""" + FROM {self.db_clause}user""" def get_sync_logs_query(self) -> str: - return f""" + return """ SELECT connector_id, sync_id, message_event, message_data, time_stamp - FROM {self.db_clause}log - WHERE message_event in ('sync_start', 'sync_end')""" + FROM {db_clause}log + WHERE message_event in ('sync_start', 'sync_end') + and time_stamp > CURRENT_TIMESTAMP - INTERVAL '{syncs_interval} days'""" def get_table_lineage_query(self) -> str: return f""" diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py index b16287dcfccb4..8843a0ad8eae6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py @@ -1180,6 +1180,7 @@ def _generate_single_profile( if custom_sql is not None: ge_config["query"] = custom_sql + batch = None with self._ge_context() as ge_context, PerfTimer() as timer: try: logger.info(f"Profiling {pretty_name}") @@ -1215,11 +1216,25 @@ def _generate_single_profile( except Exception as e: if not self.config.catch_exceptions: raise e - logger.exception(f"Encountered exception while profiling {pretty_name}") - self.report.report_warning(pretty_name, f"Profiling exception {e}") + + error_message = str(e).lower() + if "permission denied" in error_message: + self.report.warning( + title="Unauthorized to extract data profile statistics", + message="We were denied access while attempting to generate profiling statistics for some assets. Please ensure the provided user has permission to query these tables and views.", + context=f"Asset: {pretty_name}", + exc=e, + ) + else: + self.report.warning( + title="Failed to extract statistics for some assets", + message="Caught unexpected exception while attempting to extract profiling statistics for some assets.", + context=f"Asset: {pretty_name}", + exc=e, + ) return None finally: - if self.base_engine.engine.name == TRINO: + if batch is not None and self.base_engine.engine.name == TRINO: self._drop_trino_temp_table(batch) def _get_ge_dataset( diff --git a/metadata-ingestion/src/datahub/ingestion/source/git/git_import.py b/metadata-ingestion/src/datahub/ingestion/source/git/git_import.py index 2122374c1e404..6440ec8e5d787 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/git/git_import.py +++ b/metadata-ingestion/src/datahub/ingestion/source/git/git_import.py @@ -21,6 +21,8 @@ def __init__(self, tmp_dir: str, skip_known_host_verification: bool = True): def clone( self, ssh_key: Optional[SecretStr], repo_url: str, branch: Optional[str] = None ) -> Path: + # Note: this does a shallow clone. + unique_dir = str(uuid4()) keys_dir = f"{self.tmp_dir}/{unique_dir}/keys" checkout_dir = f"{self.tmp_dir}/{unique_dir}/checkout" @@ -55,20 +57,33 @@ def clone( ) logger.debug(f"ssh_command={git_ssh_cmd}") - logger.info( - f"⏳ Cloning repo '{self.sanitize_repo_url(repo_url)}', this can take some time..." - ) - self.last_repo_cloned = git.Repo.clone_from( - repo_url, - checkout_dir, - env=dict(GIT_SSH_COMMAND=git_ssh_cmd), - ) - logger.info("✅ Cloning complete!") - - if branch is not None: + if branch is None: + logger.info( + f"⏳ Cloning repo '{self.sanitize_repo_url(repo_url)}' (default branch), this can take some time..." + ) + self.last_repo_cloned = git.Repo.clone_from( + repo_url, + checkout_dir, + env=dict(GIT_SSH_COMMAND=git_ssh_cmd), + depth=1, + ) + else: + # Because we accept branch names, tags, and commit hashes in the branch parameter, + # we can't just use the --branch flag of Git clone. Doing a blobless clone allows + # us to quickly checkout the right commit. + logger.info( + f"⏳ Cloning repo '{self.sanitize_repo_url(repo_url)}' (branch: {branch}), this can take some time..." + ) + self.last_repo_cloned = git.Repo.clone_from( + repo_url, + checkout_dir, + env=dict(GIT_SSH_COMMAND=git_ssh_cmd), + filter="blob:none", + ) logger.info(f"Checking out branch {branch}") self.last_repo_cloned.git.checkout(branch) + logger.info("✅ Cloning complete!") return pathlib.Path(checkout_dir) def get_last_repo_cloned(self) -> Optional[git.Repo]: diff --git a/metadata-ingestion/src/datahub/ingestion/source/grafana/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/grafana/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_source.py b/metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_source.py new file mode 100644 index 0000000000000..53f71046c25c0 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_source.py @@ -0,0 +1,131 @@ +from typing import Iterable, List, Optional + +import requests +from pydantic import Field, SecretStr + +import datahub.emitter.mce_builder as builder +from datahub.configuration.source_common import PlatformInstanceConfigMixin +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.api.decorators import ( + SupportStatus, + config_class, + platform_name, + support_status, +) +from datahub.ingestion.api.source import MetadataWorkUnitProcessor +from datahub.ingestion.api.source_helpers import auto_workunit +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalHandler, + StaleEntityRemovalSourceReport, + StatefulIngestionConfigBase, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionReport, + StatefulIngestionSourceBase, +) +from datahub.metadata.com.linkedin.pegasus2avro.common import ChangeAuditStamps +from datahub.metadata.schema_classes import DashboardInfoClass, StatusClass + + +class GrafanaSourceConfig(StatefulIngestionConfigBase, PlatformInstanceConfigMixin): + url: str = Field( + default="", + description="Grafana URL in the format http://your-grafana-instance with no trailing slash", + ) + service_account_token: SecretStr = Field( + description="Service account token for Grafana" + ) + + +class GrafanaReport(StaleEntityRemovalSourceReport): + pass + + +@platform_name("Grafana") +@config_class(GrafanaSourceConfig) +@support_status(SupportStatus.TESTING) +class GrafanaSource(StatefulIngestionSourceBase): + """ + This is an experimental source for Grafana. + Currently only ingests dashboards (no charts) + """ + + def __init__(self, config: GrafanaSourceConfig, ctx: PipelineContext): + super().__init__(config, ctx) + self.source_config = config + self.report = GrafanaReport() + self.platform = "grafana" + + @classmethod + def create(cls, config_dict, ctx): + config = GrafanaSourceConfig.parse_obj(config_dict) + return cls(config, ctx) + + def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: + return [ + *super().get_workunit_processors(), + StaleEntityRemovalHandler.create( + self, self.source_config, self.ctx + ).workunit_processor, + ] + + def get_report(self) -> StatefulIngestionReport: + return self.report + + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: + headers = { + "Authorization": f"Bearer {self.source_config.service_account_token.get_secret_value()}", + "Content-Type": "application/json", + } + try: + response = requests.get( + f"{self.source_config.url}/api/search", headers=headers + ) + response.raise_for_status() + except requests.exceptions.RequestException as e: + self.report.report_failure(f"Failed to fetch dashboards: {str(e)}") + return + res_json = response.json() + for item in res_json: + uid = item["uid"] + title = item["title"] + url_path = item["url"] + full_url = f"{self.source_config.url}{url_path}" + dashboard_urn = builder.make_dashboard_urn( + platform=self.platform, + name=uid, + platform_instance=self.source_config.platform_instance, + ) + + yield from auto_workunit( + MetadataChangeProposalWrapper.construct_many( + entityUrn=dashboard_urn, + aspects=[ + DashboardInfoClass( + description="", + title=title, + charts=[], + lastModified=ChangeAuditStamps(), + externalUrl=full_url, + customProperties={ + key: str(value) + for key, value in { + "displayName": title, + "id": item["id"], + "uid": uid, + "title": title, + "uri": item["uri"], + "type": item["type"], + "folderId": item.get("folderId"), + "folderUid": item.get("folderUid"), + "folderTitle": item.get("folderTitle"), + }.items() + if value is not None + }, + ), + StatusClass(removed=False), + ], + ) + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py index 2585260434a38..b5caa83b2ff37 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py +++ b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py @@ -4,6 +4,7 @@ from typing import Any, Dict, Iterable, List, Optional from pyiceberg.catalog import Catalog +from pyiceberg.exceptions import NoSuchIcebergTableError from pyiceberg.schema import Schema, SchemaVisitorPerPrimitiveType, visit from pyiceberg.table import Table from pyiceberg.typedef import Identifier @@ -76,6 +77,9 @@ ) LOGGER = logging.getLogger(__name__) +logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel( + logging.WARNING +) @platform_name("Iceberg") @@ -134,9 +138,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: catalog = self.config.get_catalog() except Exception as e: LOGGER.error("Failed to get catalog", exc_info=True) - self.report.report_failure( - "get-catalog", f"Failed to get catalog {self.config.catalog.name}: {e}" - ) + self.report.report_failure("get-catalog", f"Failed to get catalog: {e}") return for dataset_path in self._get_datasets(catalog): @@ -150,7 +152,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: # Try to load an Iceberg table. Might not contain one, this will be caught by NoSuchIcebergTableError. table = catalog.load_table(dataset_path) yield from self._create_iceberg_workunit(dataset_name, table) - except Exception as e: + except NoSuchIcebergTableError as e: self.report.report_failure("general", f"Failed to create workunit: {e}") LOGGER.exception( f"Exception while processing table {dataset_path}, skipping it.", @@ -175,6 +177,7 @@ def _create_iceberg_workunit( custom_properties = table.metadata.properties.copy() custom_properties["location"] = table.metadata.location custom_properties["format-version"] = str(table.metadata.format_version) + custom_properties["partition-spec"] = str(self._get_partition_aspect(table)) if table.current_snapshot(): custom_properties["snapshot-id"] = str(table.current_snapshot().snapshot_id) custom_properties["manifest-list"] = table.current_snapshot().manifest_list @@ -204,6 +207,49 @@ def _create_iceberg_workunit( profiler = IcebergProfiler(self.report, self.config.profiling) yield from profiler.profile_table(dataset_name, dataset_urn, table) + def _get_partition_aspect(self, table: Table) -> Optional[str]: + """Extracts partition information from the provided table and returns a JSON array representing the [partition spec](https://iceberg.apache.org/spec/?#partition-specs) of the table. + Each element of the returned array represents a field in the [partition spec](https://iceberg.apache.org/spec/?#partition-specs) that follows [Appendix-C](https://iceberg.apache.org/spec/?#appendix-c-json-serialization) of the Iceberg specification. + Extra information has been added to this spec to make the information more user-friendly. + + Since Datahub does not have a place in its model to store this information, it is saved as a JSON string and displayed as a table property. + + Here is an example: + ```json + "partition-spec": "[{\"name\": \"timeperiod_loaded\", \"transform\": \"identity\", \"source\": \"timeperiod_loaded\", \"source-id\": 19, \"source-type\": \"date\", \"field-id\": 1000}]", + ``` + + Args: + table (Table): The Iceberg table to extract partition spec from. + + Returns: + str: JSON representation of the partition spec of the provided table (empty array if table is not partitioned) or `None` if an error occured. + """ + try: + return json.dumps( + [ + { + "name": partition.name, + "transform": str(partition.transform), + "source": str( + table.schema().find_column_name(partition.source_id) + ), + "source-id": partition.source_id, + "source-type": str( + table.schema().find_type(partition.source_id) + ), + "field-id": partition.field_id, + } + for partition in table.spec().fields + ] + ) + except Exception as e: + self.report.report_warning( + "extract-partition", + f"Failed to extract partition spec from Iceberg table {table.name()} due to error: {str(e)}", + ) + return None + def _get_ownership_aspect(self, table: Table) -> Optional[OwnershipClass]: owners = [] if self.config.user_ownership_property: @@ -432,6 +478,25 @@ def visit_timestamp(self, timestamp_type: TimestampType) -> Dict[str, Any]: "native_data_type": str(timestamp_type), } + # visit_timestamptz() is required when using pyiceberg >= 0.5.0, which is essentially a duplicate + # of visit_timestampz(). The function has been renamed from visit_timestampz(). + # Once Datahub can upgrade its pyiceberg dependency to >=0.5.0, the visit_timestampz() function can be safely removed. + def visit_timestamptz(self, timestamptz_type: TimestamptzType) -> Dict[str, Any]: + # Avro supports 2 types of timestamp: + # - Timestamp: independent of a particular timezone or calendar (TZ information is lost) + # - Local Timestamp: represents a timestamp in a local timezone, regardless of what specific time zone is considered local + # utcAdjustment: bool = True + return { + "type": "long", + "logicalType": "timestamp-micros", + # Commented out since Avro's Python implementation (1.11.0) does not support local-timestamp-micros, even though it exists in the spec. + # See bug report: https://issues.apache.org/jira/browse/AVRO-3476 and PR https://github.com/apache/avro/pull/1634 + # "logicalType": "timestamp-micros" + # if timestamp_type.adjust_to_utc + # else "local-timestamp-micros", + "native_data_type": str(timestamptz_type), + } + def visit_timestampz(self, timestamptz_type: TimestamptzType) -> Dict[str, Any]: # Avro supports 2 types of timestamp: # - Timestamp: independent of a particular timezone or calendar (TZ information is lost) diff --git a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py index f4d93f67b27af..b74c096d0798e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py @@ -1,7 +1,8 @@ +import logging from dataclasses import dataclass, field -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional -from pydantic import Field +from pydantic import Field, validator from pyiceberg.catalog import Catalog, load_catalog from datahub.configuration.common import AllowDenyPattern, ConfigModel @@ -18,6 +19,8 @@ is_profiling_enabled, ) +logger = logging.getLogger(__name__) + class IcebergProfilingConfig(ConfigModel): enabled: bool = Field( @@ -50,32 +53,14 @@ class IcebergProfilingConfig(ConfigModel): # include_field_sample_values: bool = True -class IcebergCatalogConfig(ConfigModel): - """ - Iceberg catalog config. - - https://py.iceberg.apache.org/configuration/ - """ - - name: str = Field( - default="default", - description="Name of catalog", - ) - type: str = Field( - description="Type of catalog. See [PyIceberg](https://py.iceberg.apache.org/configuration/) for list of possible values.", - ) - config: Dict[str, str] = Field( - description="Catalog specific configuration. See [PyIceberg documentation](https://py.iceberg.apache.org/configuration/) for details.", - ) - - class IcebergSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin): # Override the stateful_ingestion config param with the Iceberg custom stateful ingestion config in the IcebergSourceConfig stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = Field( default=None, description="Iceberg Stateful Ingestion Config." ) - catalog: IcebergCatalogConfig = Field( - description="Catalog configuration where to find Iceberg tables. See [pyiceberg's catalog configuration details](https://py.iceberg.apache.org/configuration/).", + # The catalog configuration is using a dictionary to be open and flexible. All the keys and values are handled by pyiceberg. This will future-proof any configuration change done by pyiceberg. + catalog: Dict[str, Dict[str, Any]] = Field( + description="Catalog configuration where to find Iceberg tables. Only one catalog specification is supported. The format is the same as [pyiceberg's catalog configuration](https://py.iceberg.apache.org/configuration/), where the catalog name is specified as the object name and attributes are set as key-value pairs.", ) table_pattern: AllowDenyPattern = Field( default=AllowDenyPattern.allow_all(), @@ -91,6 +76,45 @@ class IcebergSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin) ) profiling: IcebergProfilingConfig = IcebergProfilingConfig() + @validator("catalog", pre=True, always=True) + def handle_deprecated_catalog_format(cls, value): + # Once support for deprecated format is dropped, we can remove this validator. + if ( + isinstance(value, dict) + and "name" in value + and "type" in value + and "config" in value + ): + # This looks like the deprecated format + logger.warning( + "The catalog configuration format you are using is deprecated and will be removed in a future version. Please update to the new format.", + ) + catalog_name = value["name"] + catalog_type = value["type"] + catalog_config = value["config"] + new_catalog_config = { + catalog_name: {"type": catalog_type, **catalog_config} + } + return new_catalog_config + # In case the input is already the new format or is invalid + return value + + @validator("catalog") + def validate_catalog_size(cls, value): + if len(value) != 1: + raise ValueError("The catalog must contain exactly one entry.") + + # Retrieve the dict associated with the one catalog entry + catalog_name, catalog_config = next(iter(value.items())) + + # Check if that dict is not empty + if not catalog_config or not isinstance(catalog_config, dict): + raise ValueError( + f"The catalog configuration for '{catalog_name}' must not be empty and should be a dictionary with at least one key-value pair." + ) + + return value + def is_profiling_enabled(self) -> bool: return self.profiling.enabled and is_profiling_enabled( self.profiling.operation_config @@ -102,9 +126,12 @@ def get_catalog(self) -> Catalog: Returns: Catalog: Iceberg catalog instance. """ - return load_catalog( - name=self.catalog.name, **{"type": self.catalog.type, **self.catalog.config} - ) + if not self.catalog: + raise ValueError("No catalog configuration found") + + # Retrieve the dict associated with the one catalog entry + catalog_name, catalog_config = next(iter(self.catalog.items())) + return load_catalog(name=catalog_name, **catalog_config) @dataclass diff --git a/metadata-ingestion/src/datahub/ingestion/source/identity/azure_ad.py b/metadata-ingestion/src/datahub/ingestion/source/identity/azure_ad.py index 2bd05ca11e234..885b6514779cc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/identity/azure_ad.py +++ b/metadata-ingestion/src/datahub/ingestion/source/identity/azure_ad.py @@ -9,6 +9,7 @@ import click import requests from pydantic.fields import Field +from requests.adapters import HTTPAdapter, Retry from datahub.configuration.common import AllowDenyPattern from datahub.configuration.source_common import DatasetSourceConfigMixin @@ -268,6 +269,14 @@ def __init__(self, config: AzureADConfig, ctx: PipelineContext): self.report = AzureADSourceReport( filtered_tracking=self.config.filtered_tracking ) + session = requests.Session() + retries = Retry( + total=5, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504] + ) + adapter = HTTPAdapter(max_retries=retries) + session.mount("http://", adapter) + session.mount("https://", adapter) + self.session = session self.token_data = { "grant_type": "client_credentials", "client_id": self.config.client_id, @@ -494,7 +503,7 @@ def _get_azure_ad_data(self, kind: str) -> Iterable[List]: while True: if not url: break - response = requests.get(url, headers=headers) + response = self.session.get(url, headers=headers) if response.status_code == 200: json_data = json.loads(response.text) try: @@ -505,13 +514,14 @@ def _get_azure_ad_data(self, kind: str) -> Iterable[List]: yield json_data["value"] else: error_str = ( + f"Request URL: {url}. " f"Response status code: {str(response.status_code)}. " f"Response content: {str(response.content)}" ) logger.debug(f"URL = {url}") logger.error(error_str) self.report.report_failure("_get_azure_ad_data_", error_str) - continue + raise Exception(f"Unable to get {url}, error {response.status_code}") def _map_identity_to_urn(self, func, id_to_extract, mapping_identifier, id_type): result, error_str = None, None diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka.py index 99ef737206ab0..0d718e509d5c5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/kafka.py +++ b/metadata-ingestion/src/datahub/ingestion/source/kafka.py @@ -303,34 +303,63 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: ).topics extra_topic_details = self.fetch_extra_topic_details(topics.keys()) - for t, t_detail in topics.items(): - self.report.report_topic_scanned(t) - if self.source_config.topic_patterns.allowed(t): + for topic, topic_detail in topics.items(): + self.report.report_topic_scanned(topic) + if self.source_config.topic_patterns.allowed(topic): try: yield from self._extract_record( - t, t_detail, extra_topic_details.get(t) + topic, False, topic_detail, extra_topic_details.get(topic) ) except Exception as e: - logger.warning(f"Failed to extract topic {t}", exc_info=True) + logger.warning(f"Failed to extract topic {topic}", exc_info=True) self.report.report_warning( - "topic", f"Exception while extracting topic {t}: {e}" + "topic", f"Exception while extracting topic {topic}: {e}" ) else: - self.report.report_dropped(t) + self.report.report_dropped(topic) + + # Get all subjects from schema registry and ingest them as SCHEMA DatasetSubTypes + for subject in self.schema_registry_client.get_subjects(): + try: + yield from self._extract_record( + subject, True, topic_detail=None, extra_topic_config=None + ) + except Exception as e: + logger.warning(f"Failed to extract subject {subject}", exc_info=True) + self.report.report_warning( + "subject", f"Exception while extracting topic {subject}: {e}" + ) def _extract_record( self, topic: str, + is_subject: bool, topic_detail: Optional[TopicMetadata], extra_topic_config: Optional[Dict[str, ConfigEntry]], ) -> Iterable[MetadataWorkUnit]: - logger.debug(f"topic = {topic}") - AVRO = "AVRO" - # 1. Create the default dataset snapshot for the topic. - dataset_name = topic + kafka_entity = "subject" if is_subject else "topic" + + logger.debug(f"extracting schema metadata from kafka entity = {kafka_entity}") + platform_urn = make_data_platform_urn(self.platform) + + # 1. Create schemaMetadata aspect (pass control to SchemaRegistry) + schema_metadata = self.schema_registry_client.get_schema_metadata( + topic, platform_urn, is_subject + ) + + # topic can have no associated subject, but still it can be ingested without schema + # for schema ingestion, ingest only if it has valid schema + if is_subject: + if schema_metadata is None: + return + dataset_name = schema_metadata.schemaName + else: + dataset_name = topic + + # 2. Create the default dataset snapshot for the topic. dataset_urn = make_dataset_urn_with_platform_instance( platform=self.platform, name=dataset_name, @@ -342,10 +371,6 @@ def _extract_record( aspects=[Status(removed=False)], # we append to this list later on ) - # 2. Attach schemaMetadata aspect (pass control to SchemaRegistry) - schema_metadata = self.schema_registry_client.get_schema_metadata( - topic, platform_urn - ) if schema_metadata is not None: dataset_snapshot.aspects.append(schema_metadata) @@ -356,9 +381,19 @@ def _extract_record( browse_path = BrowsePathsClass([browse_path_str]) dataset_snapshot.aspects.append(browse_path) - custom_props = self.build_custom_properties( - topic, topic_detail, extra_topic_config - ) + # build custom properties for topic, schema properties may be added as needed + custom_props: Dict[str, str] = {} + if not is_subject: + custom_props = self.build_custom_properties( + topic, topic_detail, extra_topic_config + ) + schema_name: Optional[ + str + ] = self.schema_registry_client._get_subject_for_topic( + topic, is_key_schema=False + ) + if schema_name is not None: + custom_props["Schema Name"] = schema_name # 4. Set dataset's description, tags, ownership, etc, if topic schema type is avro description: Optional[str] = None @@ -414,7 +449,7 @@ def _extract_record( ) dataset_properties = DatasetPropertiesClass( - name=topic, customProperties=custom_props, description=description + name=dataset_name, customProperties=custom_props, description=description ) dataset_snapshot.aspects.append(dataset_properties) @@ -431,12 +466,13 @@ def _extract_record( # 6. Emit the datasetSnapshot MCE mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) - yield MetadataWorkUnit(id=f"kafka-{topic}", mce=mce) + yield MetadataWorkUnit(id=f"kafka-{kafka_entity}", mce=mce) - # 7. Add the subtype aspect marking this as a "topic" + # 7. Add the subtype aspect marking this as a "topic" or "schema" + typeName = DatasetSubTypes.SCHEMA if is_subject else DatasetSubTypes.TOPIC yield MetadataChangeProposalWrapper( entityUrn=dataset_urn, - aspect=SubTypesClass(typeNames=[DatasetSubTypes.TOPIC]), + aspect=SubTypesClass(typeNames=[typeName]), ).as_workunit() domain_urn: Optional[str] = None diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py index cf70eb95762c4..17047457e0eba 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py +++ b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py @@ -773,6 +773,7 @@ class BQParser: project: str target_platform: str sanitizeTopics: str + transforms: list topicsToTables: Optional[str] = None datasets: Optional[str] = None defaultDataset: Optional[str] = None @@ -788,6 +789,20 @@ def get_parser( ) -> BQParser: project = connector_manifest.config["project"] sanitizeTopics = connector_manifest.config.get("sanitizeTopics", "false") + transform_names = ( + self.connector_manifest.config.get("transforms", "").split(",") + if self.connector_manifest.config.get("transforms") + else [] + ) + transforms = [] + for name in transform_names: + transform = {"name": name} + transforms.append(transform) + for key in self.connector_manifest.config.keys(): + if key.startswith(f"transforms.{name}."): + transform[ + key.replace(f"transforms.{name}.", "") + ] = self.connector_manifest.config[key] if "defaultDataset" in connector_manifest.config: defaultDataset = connector_manifest.config["defaultDataset"] @@ -797,6 +812,7 @@ def get_parser( target_platform="bigquery", sanitizeTopics=sanitizeTopics.lower() == "true", version="v2", + transforms=transforms, ) else: # version 1.6.x and similar configs supported @@ -809,6 +825,7 @@ def get_parser( datasets=datasets, target_platform="bigquery", sanitizeTopics=sanitizeTopics.lower() == "true", + transforms=transforms, ) def get_list(self, property: str) -> Iterable[Tuple[str, str]]: @@ -867,6 +884,18 @@ def get_dataset_table_for_topic( table = self.sanitize_table_name(table) return f"{dataset}.{table}" + def apply_transformations( + self, topic: str, transforms: List[Dict[str, str]] + ) -> str: + for transform in transforms: + if transform["type"] == "org.apache.kafka.connect.transforms.RegexRouter": + regex = transform["regex"] + replacement = transform["replacement"] + pattern = re.compile(regex) + if pattern.match(topic): + topic = pattern.sub(replacement, topic, count=1) + return topic + def _extract_lineages(self): lineages: List[KafkaConnectLineage] = list() parser = self.get_parser(self.connector_manifest) @@ -874,26 +903,26 @@ def _extract_lineages(self): return lineages target_platform = parser.target_platform project = parser.project - + transforms = parser.transforms self.connector_manifest.flow_property_bag = self.connector_manifest.config - # Mask/Remove properties that may reveal credentials if "keyfile" in self.connector_manifest.flow_property_bag: del self.connector_manifest.flow_property_bag["keyfile"] for topic in self.connector_manifest.topic_names: - dataset_table = self.get_dataset_table_for_topic(topic, parser) + transformed_topic = self.apply_transformations(topic, transforms) + dataset_table = self.get_dataset_table_for_topic(transformed_topic, parser) if dataset_table is None: self.report_warning( self.connector_manifest.name, - f"could not find target dataset for topic {topic}, please check your connector configuration", + f"could not find target dataset for topic {transformed_topic}, please check your connector configuration", ) continue target_dataset = f"{project}.{dataset_table}" lineages.append( KafkaConnectLineage( - source_dataset=topic, + source_dataset=transformed_topic, source_platform=KAFKA, target_dataset=target_dataset, target_platform=target_platform, diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_schema_registry_base.py b/metadata-ingestion/src/datahub/ingestion/source/kafka_schema_registry_base.py index 34ff76f44d1dd..59f174a9a5045 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/kafka_schema_registry_base.py +++ b/metadata-ingestion/src/datahub/ingestion/source/kafka_schema_registry_base.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Optional +from typing import List, Optional from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaMetadata @@ -7,6 +7,16 @@ class KafkaSchemaRegistryBase(ABC): @abstractmethod def get_schema_metadata( - self, topic: str, platform_urn: str + self, topic: str, platform_urn: str, is_subject: bool ) -> Optional[SchemaMetadata]: pass + + @abstractmethod + def get_subjects(self) -> List[str]: + pass + + @abstractmethod + def _get_subject_for_topic( + self, dataset_subtype: str, is_key_schema: bool + ) -> Optional[str]: + pass diff --git a/metadata-ingestion/src/datahub/ingestion/source/ldap.py b/metadata-ingestion/src/datahub/ingestion/source/ldap.py index 1368a5b83fe6f..9c7fba68f263b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ldap.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ldap.py @@ -334,10 +334,12 @@ def handle_user(self, dn: str, attrs: Dict[str, Any]) -> Iterable[MetadataWorkUn manager_ldap = guess_person_ldap(m_attrs, self.config, self.report) m_email = get_attr_or_none( - m_attrs, self.config.user_attrs_map["email"], manager_ldap + m_attrs, self.config.user_attrs_map["email"] ) make_manager_urn = ( - m_email if self.config.use_email_as_username else manager_ldap + m_email + if m_email and self.config.use_email_as_username + else manager_ldap ) except ldap.LDAPError as e: @@ -377,7 +379,7 @@ def build_corp_user_mce( last_name = attrs[self.config.user_attrs_map["lastName"]][0].decode() groups = parse_groups(attrs, self.config.user_attrs_map["memberOf"]) - email = get_attr_or_none(attrs, self.config.user_attrs_map["email"], ldap_user) + email = get_attr_or_none(attrs, self.config.user_attrs_map["email"]) display_name = get_attr_or_none( attrs, self.config.user_attrs_map["displayName"], full_name ) @@ -404,7 +406,9 @@ def build_corp_user_mce( manager_urn = f"urn:li:corpuser:{manager_ldap}" if manager_ldap else None - make_user_urn = email if self.config.use_email_as_username else ldap_user + make_user_urn = ( + email if email and self.config.use_email_as_username else ldap_user + ) user_snapshot = CorpUserSnapshotClass( urn=f"urn:li:corpuser:{make_user_urn}", @@ -438,9 +442,7 @@ def build_corp_group_mce(self, attrs: dict) -> Optional[MetadataChangeEvent]: admins = parse_users(attrs, self.config.group_attrs_map["admins"]) members = parse_users(attrs, self.config.group_attrs_map["members"]) - email = get_attr_or_none( - attrs, self.config.group_attrs_map["email"], full_name - ) + email = get_attr_or_none(attrs, self.config.group_attrs_map["email"]) description = get_attr_or_none( attrs, self.config.group_attrs_map["description"] ) @@ -448,7 +450,9 @@ def build_corp_group_mce(self, attrs: dict) -> Optional[MetadataChangeEvent]: attrs, self.config.group_attrs_map["displayName"] ) - make_group_urn = email if self.config.use_email_as_username else full_name + make_group_urn = ( + email if email and self.config.use_email_as_username else full_name + ) group_snapshot = CorpGroupSnapshotClass( urn=f"urn:li:corpGroup:{make_group_urn}", diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py index 09105b2c6bfb0..1f54767de5a68 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py @@ -8,7 +8,6 @@ from enum import Enum from functools import lru_cache from typing import ( - TYPE_CHECKING, Dict, Iterable, Iterator, @@ -38,12 +37,20 @@ from datahub.ingestion.source.common.subtypes import DatasetSubTypes from datahub.ingestion.source.looker.looker_config import ( LookerCommonConfig, + LookerConnectionDefinition, LookerDashboardSourceConfig, NamingPatternMapping, ViewNamingPatternMapping, ) from datahub.ingestion.source.looker.looker_constant import IMPORTED_PROJECTS +from datahub.ingestion.source.looker.looker_dataclasses import ProjectInclude +from datahub.ingestion.source.looker.looker_file_loader import LookerViewFileLoader from datahub.ingestion.source.looker.looker_lib_wrapper import LookerAPI +from datahub.ingestion.source.looker.lookml_config import ( + _BASE_PROJECT_NAME, + LookMLSourceReport, +) +from datahub.ingestion.source.looker.str_functions import remove_suffix from datahub.ingestion.source.sql.sql_types import ( POSTGRES_TYPES_MAP, SNOWFLAKE_TYPES_MAP, @@ -93,17 +100,12 @@ TagSnapshotClass, ) from datahub.metadata.urns import TagUrn +from datahub.sql_parsing.sqlglot_lineage import ColumnRef from datahub.utilities.lossy_collections import LossyList, LossySet from datahub.utilities.url_util import remove_port_from_url CORPUSER_DATAHUB = "urn:li:corpuser:datahub" -if TYPE_CHECKING: - from datahub.ingestion.source.looker.lookml_source import ( - LookerViewFileLoader, - LookMLSourceReport, - ) - logger = logging.getLogger(__name__) @@ -126,11 +128,59 @@ class LookerFolderKey(ContainerKey): folder_id: str -def remove_suffix(original: str, suffix: str) -> str: - # This can be removed in favour of original.removesuffix for python>3.8 - if original.endswith(suffix): - return original[: -len(suffix)] - return original +def deduplicate_fields(fields: List["ViewField"]) -> List["ViewField"]: + # Remove duplicates filed from self.fields + # Logic is: If more than a field has same ViewField.name then keep only one filed where ViewField.field_type + # is DIMENSION_GROUP. + # Looker Constraint: + # - Any field declared as dimension or measure can be redefined as dimension_group. + # - Any field declared in dimension can't be redefined in measure and vice-versa. + + dimension_group_field_names: List[str] = [ + field.name + for field in fields + if field.field_type == ViewFieldType.DIMENSION_GROUP + ] + + new_fields: List[ViewField] = [] + + for field in fields: + if ( + field.name in dimension_group_field_names + and field.field_type != ViewFieldType.DIMENSION_GROUP + ): + continue + + new_fields.append(field) + + return new_fields + + +def find_view_from_resolved_includes( + connection: Optional[LookerConnectionDefinition], + resolved_includes: List["ProjectInclude"], + looker_viewfile_loader: LookerViewFileLoader, + target_view_name: str, + reporter: LookMLSourceReport, +) -> Optional[Tuple["ProjectInclude", dict]]: + # It could live in one of the included files. We do not know which file the base view + # lives in, so we try them all! + for include in resolved_includes: + included_looker_viewfile = looker_viewfile_loader.load_viewfile( + include.include, + include.project, + connection, + reporter, + ) + if not included_looker_viewfile: + continue + for raw_view in included_looker_viewfile.views: + raw_view_name = raw_view["name"] + # Make sure to skip loading view we are currently trying to resolve + if raw_view_name == target_view_name: + return include, raw_view + + return None @dataclass @@ -243,7 +293,46 @@ class ViewField: project_name: Optional[str] = None view_name: Optional[str] = None is_primary_key: bool = False - upstream_fields: List[str] = dataclasses_field(default_factory=list) + tags: List[str] = dataclasses_field(default_factory=list) + + # It is the list of ColumnRef for derived view defined using SQL otherwise simple column name + upstream_fields: Union[List[ColumnRef]] = dataclasses_field(default_factory=list) + + @classmethod + def view_fields_from_dict( + cls, + field_dict: Dict, + upstream_column_ref: List[ColumnRef], + type_cls: ViewFieldType, + populate_sql_logic_in_descriptions: bool, + ) -> "ViewField": + + is_primary_key = field_dict.get("primary_key", "no") == "yes" + + name = field_dict["name"] + + native_type = field_dict.get("type", "string") + + default_description = ( + f"sql:{field_dict['sql']}" + if "sql" in field_dict and populate_sql_logic_in_descriptions + else "" + ) + + description = field_dict.get("description", default_description) + + label = field_dict.get("label", "") + + return ViewField( + name=name, + type=native_type, + label=label, + description=description, + is_primary_key=is_primary_key, + field_type=type_cls, + tags=field_dict.get("tags") or [], + upstream_fields=upstream_column_ref, + ) @dataclass @@ -251,49 +340,109 @@ class ExploreUpstreamViewField: explore: LookmlModelExplore field: LookmlModelExploreField - def _form_field_name(self): + def _form_field_name( + self, + view_project_map: Dict[str, str], + explore_project_name: str, + model_name: str, + upstream_views_file_path: Dict[str, Optional[str]], + config: LookerCommonConfig, + remove_variant: bool = False, + ) -> Optional[ColumnRef]: assert self.field.name is not None if len(self.field.name.split(".")) != 2: - return self.field.name # Inconsistent info received + return None # Inconsistent info received view_name: Optional[str] = self.explore.name - - if ( - self.field.original_view is not None - ): # if `from` is used in explore then original_view is pointing to - # lookml view + if self.field.original_view is not None: view_name = self.field.original_view field_name = self.field.name.split(".")[1] - return f"{view_name}.{field_name}" + if remove_variant and self.field.field_group_variant is not None: + # remove variant at the end. +1 for "_" + field_name = field_name[ + : -(len(self.field.field_group_variant.lower()) + 1) + ] + + assert view_name # for lint false positive - def upstream(self) -> str: - assert self.field.name is not None + project_include: ProjectInclude = ProjectInclude( + project=view_project_map.get(view_name, _BASE_PROJECT_NAME), + include=view_name, + ) + + file_path: Optional[str] = ( + upstream_views_file_path.get(view_name) + if upstream_views_file_path.get(view_name) + else ViewFieldValue.NOT_AVAILABLE.value + ) - if self.field.dimension_group is None: # It is not part of Dimensional Group - return self._form_field_name() + assert file_path - if self.field.field_group_variant is None: - return ( - self._form_field_name() - ) # Variant i.e. Month, Day, Year ... is not available + view_urn = LookerViewId( + project_name=( + project_include.project + if project_include.project != _BASE_PROJECT_NAME + else explore_project_name + ), + model_name=model_name, + view_name=project_include.include, + file_path=file_path, + ).get_urn(config) + + return ColumnRef( + table=view_urn, + column=field_name, + ) + + def upstream( + self, + view_project_map: Dict[str, str], + explore_project_name: str, + model_name: str, + upstream_views_file_path: Dict[str, Optional[str]], + config: LookerCommonConfig, + ) -> Optional[ColumnRef]: + assert self.field.name is not None + + if self.field.dimension_group is None or self.field.field_group_variant is None: + return self._form_field_name( + view_project_map, + explore_project_name, + model_name, + upstream_views_file_path, + config, + ) if self.field.type is None or not self.field.type.startswith("date_"): - return ( - self._form_field_name() + return self._form_field_name( + view_project_map, + explore_project_name, + model_name, + upstream_views_file_path, + config, ) # for Dimensional Group the type is always start with date_[time|date] if not self.field.name.endswith(f"_{self.field.field_group_variant.lower()}"): - return ( - self._form_field_name() + return self._form_field_name( + view_project_map, + explore_project_name, + model_name, + upstream_views_file_path, + config, ) # if the explore field is generated because of Dimensional Group in View # then the field_name should ends with field_group_variant - return self._form_field_name()[ - : -(len(self.field.field_group_variant.lower()) + 1) - ] # remove variant at the end. +1 for "_" + return self._form_field_name( + view_project_map, + explore_project_name, + model_name, + upstream_views_file_path, + config, + remove_variant=True, + ) def create_view_project_map(view_fields: List[ViewField]) -> Dict[str, str]: @@ -480,9 +629,7 @@ def extract_project_name_from_source_file( return None @staticmethod - def _get_field_type( - native_type: str, reporter: SourceReport - ) -> SchemaFieldDataType: + def get_field_type(native_type: str) -> SchemaFieldDataType: type_class = LookerUtil.field_type_mapping.get(native_type) if type_class is None: @@ -561,21 +708,30 @@ def _get_tag_mce_for_urn(tag_urn: str) -> MetadataChangeEvent: @staticmethod def _get_tags_from_field_type( - field_type: ViewFieldType, reporter: SourceReport + field: ViewField, reporter: SourceReport ) -> Optional[GlobalTagsClass]: - if field_type in LookerUtil.type_to_tag_map: - return GlobalTagsClass( - tags=[ + schema_field_tags: List[TagAssociationClass] = [ + TagAssociationClass(tag=builder.make_tag_urn(tag_name)) + for tag_name in field.tags + ] + + if field.field_type in LookerUtil.type_to_tag_map: + schema_field_tags.extend( + [ TagAssociationClass(tag=tag_name) - for tag_name in LookerUtil.type_to_tag_map[field_type] + for tag_name in LookerUtil.type_to_tag_map[field.field_type] ] ) else: reporter.report_warning( - "lookml", - f"Failed to map view field type {field_type}. Won't emit tags for it", + title="Failed to Map View Field Type", + message=f"Failed to map view field type {field.field_type}. Won't emit tags for measure and dimension", ) - return None + + if schema_field_tags: + return GlobalTagsClass(tags=schema_field_tags) + + return None @staticmethod def get_tag_mces() -> Iterable[MetadataChangeEvent]: @@ -593,7 +749,7 @@ def view_field_to_schema_field( ) -> SchemaField: return SchemaField( fieldPath=field.name, - type=LookerUtil._get_field_type(field.type, reporter), + type=LookerUtil.get_field_type(field.type), nativeDataType=field.type, label=field.label, description=( @@ -602,7 +758,7 @@ def view_field_to_schema_field( else f"{field.field_type.value}. {field.description}" ), globalTags=( - LookerUtil._get_tags_from_field_type(field.field_type, reporter) + LookerUtil._get_tags_from_field_type(field, reporter) if tag_measures_and_dimensions is True else None ), @@ -648,12 +804,6 @@ def create_query_request(q: dict, limit: Optional[str] = None) -> WriteQuery: ) -@dataclass(frozen=True, order=True) -class ProjectInclude: - project: str - include: str - - @dataclass class LookerExplore: name: str @@ -689,8 +839,8 @@ def from_dict( model_name: str, dict: Dict, resolved_includes: List[ProjectInclude], - looker_viewfile_loader: "LookerViewFileLoader", - reporter: "LookMLSourceReport", + looker_viewfile_loader: LookerViewFileLoader, + reporter: LookMLSourceReport, model_explores_map: Dict[str, dict], ) -> "LookerExplore": view_names: Set[str] = set() @@ -714,12 +864,6 @@ def from_dict( fields = cls._get_fields_from_sql_equality(sql_on) joins = fields - # HACK: We shouldn't be doing imports here. We also have - # circular imports that don't belong. - from datahub.ingestion.source.looker.lookml_source import ( - _find_view_from_resolved_includes, - ) - upstream_views: List[ProjectInclude] = [] # create the list of extended explores extends = list( @@ -746,7 +890,7 @@ def from_dict( else: # we only fallback to the view_names list if this is not an extended explore for view_name in view_names: - info = _find_view_from_resolved_includes( + info = find_view_from_resolved_includes( None, resolved_includes, looker_viewfile_loader, @@ -825,8 +969,9 @@ def from_api( # noqa: C901 potential_views.append(view_name) except AssertionError: reporter.report_warning( - key=f"chart-field-{field_name}", - reason="The field was not prefixed by a view name. This can happen when the field references another dynamic field.", + title="Missing View Name", + message="The field was not prefixed by a view name. This can happen when the field references another dynamic field.", + context=field_name, ) continue @@ -839,18 +984,17 @@ def from_api( # noqa: C901 views.add(view_name) view_fields: List[ViewField] = [] + field_name_vs_raw_explore_field: Dict = {} + if explore.fields is not None: + if explore.fields.dimensions is not None: for dim_field in explore.fields.dimensions: if dim_field.name is None: continue else: - dimension_upstream_field: ExploreUpstreamViewField = ( - ExploreUpstreamViewField( - explore=explore, - field=dim_field, - ) - ) + + field_name_vs_raw_explore_field[dim_field.name] = dim_field view_fields.append( ViewField( @@ -882,9 +1026,7 @@ def from_api( # noqa: C901 if dim_field.primary_key else False ), - upstream_fields=[ - dimension_upstream_field.upstream() - ], + upstream_fields=[], ) ) if explore.fields.measures is not None: @@ -892,12 +1034,10 @@ def from_api( # noqa: C901 if measure_field.name is None: continue else: - measure_upstream_field: ExploreUpstreamViewField = ( - ExploreUpstreamViewField( - explore=explore, - field=measure_field, - ) - ) + + field_name_vs_raw_explore_field[ + measure_field.name + ] = measure_field view_fields.append( ViewField( @@ -925,7 +1065,7 @@ def from_api( # noqa: C901 if measure_field.primary_key else False ), - upstream_fields=[measure_upstream_field.upstream()], + upstream_fields=[], ) ) @@ -942,6 +1082,28 @@ def from_api( # noqa: C901 if upstream_views_file_path: logger.debug(f"views and their file-paths: {upstream_views_file_path}") + # form upstream of fields as all information is now available + for view_field in view_fields: + measure_upstream_field: ExploreUpstreamViewField = ( + ExploreUpstreamViewField( + explore=explore, + field=field_name_vs_raw_explore_field[view_field.name], + ) + ) + + assert explore.project_name is not None + + column_ref: Optional[ColumnRef] = measure_upstream_field.upstream( + view_project_map=view_project_map, + explore_project_name=explore.project_name, + model_name=model, + upstream_views_file_path=upstream_views_file_path, + config=source_config, + ) + view_field.upstream_fields = ( + [column_ref] if column_ref is not None else [] + ) + return cls( name=explore_name, model_name=model, @@ -972,8 +1134,9 @@ def from_api( # noqa: C901 except AssertionError: reporter.report_warning( - key="chart-", - reason="Was unable to find dependent views for this chart", + title="Unable to find Views", + message="Encountered exception while attempting to find dependent views for this chart", + context=f"Explore: {explore_name}, Mode: {model}, Views: {views}", ) return None @@ -1054,7 +1217,8 @@ def _to_metadata_events( # noqa: C901 observed_lineage_ts = datetime.datetime.now(tz=datetime.timezone.utc) for view_ref in sorted(self.upstream_views): # set file_path to ViewFieldType.UNKNOWN if file_path is not available to keep backward compatibility - # if we raise error on file_path equal to None then existing test-cases will fail as mock data doesn't have required attributes. + # if we raise error on file_path equal to None then existing test-cases will fail as mock data + # doesn't have required attributes. file_path: str = ( cast(str, self.upstream_views_file_path[view_ref.include]) if self.upstream_views_file_path[view_ref.include] is not None @@ -1086,30 +1250,24 @@ def _to_metadata_events( # noqa: C901 fine_grained_lineages = [] if config.extract_column_level_lineage: for field in self.fields or []: - for upstream_field in field.upstream_fields: - if len(upstream_field.split(".")) >= 2: - (view_name, field_path) = upstream_field.split(".")[ - 0 - ], ".".join(upstream_field.split(".")[1:]) - assert view_name - view_urn = view_name_to_urn_map.get(view_name, "") - if view_urn: - fine_grained_lineages.append( - FineGrainedLineageClass( - upstreamType=FineGrainedLineageUpstreamType.FIELD_SET, - downstreamType=FineGrainedLineageDownstreamType.FIELD, - upstreams=[ - builder.make_schema_field_urn( - view_urn, field_path - ) - ], - downstreams=[ - builder.make_schema_field_urn( - self.get_explore_urn(config), field.name - ) - ], + for upstream_column_ref in field.upstream_fields: + fine_grained_lineages.append( + FineGrainedLineageClass( + upstreamType=FineGrainedLineageUpstreamType.FIELD_SET, + downstreamType=FineGrainedLineageDownstreamType.FIELD, + upstreams=[ + builder.make_schema_field_urn( + upstream_column_ref.table, + upstream_column_ref.column, ) - ) + ], + downstreams=[ + builder.make_schema_field_urn( + self.get_explore_urn(config), field.name + ) + ], + ) + ) upstream_lineage = UpstreamLineage( upstreams=upstreams, fineGrainedLineages=fine_grained_lineages or None @@ -1137,16 +1295,9 @@ def _to_metadata_events( # noqa: C901 ] # Add tags - explore_tag_urns: List[TagAssociationClass] = [] - for tag in self.tags: - tag_urn = TagUrn(tag) - explore_tag_urns.append(TagAssociationClass(tag_urn.urn())) - proposals.append( - MetadataChangeProposalWrapper( - entityUrn=tag_urn.urn(), - aspect=tag_urn.to_key_aspect(), - ) - ) + explore_tag_urns: List[TagAssociationClass] = [ + TagAssociationClass(tag=TagUrn(tag).urn()) for tag in self.tags + ] if explore_tag_urns: dataset_snapshot.aspects.append(GlobalTagsClass(explore_tag_urns)) @@ -1384,6 +1535,7 @@ class LookerDashboardElement: input_fields: Optional[List[InputFieldElement]] = None folder_path: Optional[str] = None # for independent looks. folder: Optional[LookerFolder] = None + owner: Optional[LookerUser] = None def url(self, base_url: str) -> str: # A dashboard element can use a look or just a raw query against an explore diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py index 8de213cfabaf0..4ad9635069b15 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py @@ -1,13 +1,14 @@ import dataclasses import os import re -from typing import Any, ClassVar, Dict, List, Optional, Union, cast +from typing import Any, ClassVar, Dict, List, Optional, Tuple, Union, cast import pydantic +from looker_sdk.sdk.api40.models import DBConnection from pydantic import Field, validator from datahub.configuration import ConfigModel -from datahub.configuration.common import AllowDenyPattern +from datahub.configuration.common import AllowDenyPattern, ConfigurationError from datahub.configuration.source_common import ( EnvConfigMixin, PlatformInstanceConfigMixin, @@ -143,6 +144,82 @@ class LookerCommonConfig(EnvConfigMixin, PlatformInstanceConfigMixin): ) +def _get_bigquery_definition( + looker_connection: DBConnection, +) -> Tuple[str, Optional[str], Optional[str]]: + platform = "bigquery" + # bigquery project ids are returned in the host field + db = looker_connection.host + schema = looker_connection.database + return platform, db, schema + + +def _get_generic_definition( + looker_connection: DBConnection, platform: Optional[str] = None +) -> Tuple[str, Optional[str], Optional[str]]: + if platform is None: + # We extract the platform from the dialect name + dialect_name = looker_connection.dialect_name + assert dialect_name is not None + # generally the first part of the dialect name before _ is the name of the platform + # versions are encoded as numbers and can be removed + # e.g. spark1 or hive2 or druid_18 + platform = re.sub(r"[0-9]+", "", dialect_name.split("_")[0]) + + assert ( + platform is not None + ), f"Failed to extract a valid platform from connection {looker_connection}" + db = looker_connection.database + schema = looker_connection.schema # ok for this to be None + return platform, db, schema + + +class LookerConnectionDefinition(ConfigModel): + platform: str + default_db: str + default_schema: Optional[str] # Optional since some sources are two-level only + platform_instance: Optional[str] = None + platform_env: Optional[str] = Field( + default=None, + description="The environment that the platform is located in. Leaving this empty will inherit defaults from " + "the top level Looker configuration", + ) + + @validator("platform_env") + def platform_env_must_be_one_of(cls, v: Optional[str]) -> Optional[str]: + if v is not None: + return EnvConfigMixin.env_must_be_one_of(v) + return v + + @validator("platform", "default_db", "default_schema") + def lower_everything(cls, v): + """We lower case all strings passed in to avoid casing issues later""" + if v is not None: + return v.lower() + + @classmethod + def from_looker_connection( + cls, looker_connection: DBConnection + ) -> "LookerConnectionDefinition": + """Dialect definitions are here: https://docs.looker.com/setup-and-management/database-config""" + extractors: Dict[str, Any] = { + "^bigquery": _get_bigquery_definition, + ".*": _get_generic_definition, + } + + if looker_connection.dialect_name is None: + raise ConfigurationError( + f"Unable to fetch a fully filled out connection for {looker_connection.name}. Please check your API permissions." + ) + for extractor_pattern, extracting_function in extractors.items(): + if re.match(extractor_pattern, looker_connection.dialect_name): + (platform, db, schema) = extracting_function(looker_connection) + return cls(platform=platform, default_db=db, default_schema=schema) + raise ConfigurationError( + f"Could not find an appropriate platform for looker_connection: {looker_connection.name} with dialect: {looker_connection.dialect_name}" + ) + + class LookerDashboardSourceConfig( LookerAPIConfig, LookerCommonConfig, @@ -179,7 +256,7 @@ class LookerDashboardSourceConfig( "ingest dashboards in the Shared folder space.", ) max_threads: int = Field( - os.cpu_count() or 40, + default_factory=lambda: os.cpu_count() or 40, description="Max parallelism for Looker API calls. Defaults to cpuCount or 40", ) external_base_url: Optional[str] = Field( diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_connection.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_connection.py new file mode 100644 index 0000000000000..2b7ce6f6da026 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_connection.py @@ -0,0 +1,55 @@ +import logging +from typing import Optional + +from looker_sdk.error import SDKError +from looker_sdk.sdk.api40.models import DBConnection + +from datahub.configuration.common import ConfigurationError +from datahub.ingestion.source.looker.looker_config import LookerConnectionDefinition +from datahub.ingestion.source.looker.looker_lib_wrapper import LookerAPI +from datahub.ingestion.source.looker.lookml_config import ( + LookMLSourceConfig, + LookMLSourceReport, +) + +logger = logging.getLogger(__name__) + + +def get_connection_def_based_on_connection_string( + connection: str, + source_config: LookMLSourceConfig, + looker_client: Optional[LookerAPI], + reporter: LookMLSourceReport, +) -> Optional[LookerConnectionDefinition]: + if source_config.connection_to_platform_map is None: + source_config.connection_to_platform_map = {} + + assert source_config.connection_to_platform_map is not None + + connection_def: Optional[LookerConnectionDefinition] = None + + if connection in source_config.connection_to_platform_map: + connection_def = source_config.connection_to_platform_map[connection] + elif looker_client: + try: + looker_connection: DBConnection = looker_client.connection(connection) + except SDKError: + logger.error( + f"Failed to retrieve connection {connection} from Looker. This usually happens when the " + f"credentials provided are not admin credentials." + ) + else: + try: + connection_def = LookerConnectionDefinition.from_looker_connection( + looker_connection + ) + + # Populate the cache (using the config map) to avoid calling looker again for this connection + source_config.connection_to_platform_map[connection] = connection_def + except ConfigurationError: + reporter.report_warning( + f"connection-{connection}", + "Failed to load connection from Looker", + ) + + return connection_def diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py new file mode 100644 index 0000000000000..adaa3c4875450 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py @@ -0,0 +1,290 @@ +import glob +import logging +import pathlib +from dataclasses import dataclass +from typing import Dict, List, Optional, Set + +from datahub.ingestion.source.looker.lkml_patched import load_lkml +from datahub.ingestion.source.looker.looker_connection import LookerConnectionDefinition +from datahub.ingestion.source.looker.lookml_config import ( + _BASE_PROJECT_NAME, + _EXPLORE_FILE_EXTENSION, + LookMLSourceReport, +) + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True, order=True) +class ProjectInclude: + project: str + include: str + + +@dataclass +class LookerField: + name: str + primary_key: str # possible values yes and no + type: str + sql: Optional[str] + + +@dataclass +class LookerModel: + connection: str + includes: List[str] + explores: List[dict] + resolved_includes: List[ProjectInclude] + + @staticmethod + def from_looker_dict( + looker_model_dict: dict, + base_project_name: str, + root_project_name: Optional[str], + base_projects_folders: Dict[str, pathlib.Path], + path: str, + reporter: LookMLSourceReport, + ) -> "LookerModel": + logger.debug(f"Loading model from {path}") + connection = looker_model_dict["connection"] + includes = looker_model_dict.get("includes", []) + resolved_includes = LookerModel.resolve_includes( + includes, + base_project_name, + root_project_name, + base_projects_folders, + path, + reporter, + seen_so_far=set(), + traversal_path=pathlib.Path(path).stem, + ) + logger.debug(f"{path} has resolved_includes: {resolved_includes}") + explores = looker_model_dict.get("explores", []) + + explore_files = [ + x.include + for x in resolved_includes + if x.include.endswith(_EXPLORE_FILE_EXTENSION) + ] + for included_file in explore_files: + try: + parsed = load_lkml(included_file) + included_explores = parsed.get("explores", []) + explores.extend(included_explores) + except Exception as e: + reporter.report_warning( + title="Error Loading Include", + message="Failed to load include file", + context=f"Include Details: {included_file}", + exc=e, + ) + # continue in this case, as it might be better to load and resolve whatever we can + + return LookerModel( + connection=connection, + includes=includes, + resolved_includes=resolved_includes, + explores=explores, + ) + + @staticmethod + def resolve_includes( + includes: List[str], + project_name: str, + root_project_name: Optional[str], + base_projects_folder: Dict[str, pathlib.Path], + path: str, + reporter: LookMLSourceReport, + seen_so_far: Set[str], + traversal_path: str = "", # a cosmetic parameter to aid debugging + ) -> List[ProjectInclude]: + """Resolve ``include`` statements in LookML model files to a list of ``.lkml`` files. + + For rules on how LookML ``include`` statements are written, see + https://docs.looker.com/data-modeling/getting-started/ide-folders#wildcard_examples + """ + + resolved = [] + for inc in includes: + # Filter out dashboards - we get those through the looker source. + if ( + inc.endswith(".dashboard") + or inc.endswith(".dashboard.lookml") + or inc.endswith(".dashboard.lkml") + ): + logger.debug(f"include '{inc}' is a dashboard, skipping it") + continue + + resolved_project_name = project_name + resolved_project_folder = str(base_projects_folder[project_name]) + + # Massage the looker include into a valid glob wildcard expression + if inc.startswith("//"): + # remote include, let's see if we have the project checked out locally + (remote_project, project_local_path) = inc[2:].split("/", maxsplit=1) + if remote_project in base_projects_folder: + resolved_project_folder = str(base_projects_folder[remote_project]) + glob_expr = f"{resolved_project_folder}/{project_local_path}" + resolved_project_name = remote_project + else: + logger.warning( + f"Resolving {inc} failed. Could not find a locally checked out reference for {remote_project}" + ) + continue + elif inc.startswith("/"): + glob_expr = f"{resolved_project_folder}{inc}" + + # The include path is sometimes '/{project_name}/{path_within_project}' + # instead of '//{project_name}/{path_within_project}' or '/{path_within_project}'. + # + # TODO: I can't seem to find any documentation on this pattern, but we definitely + # have seen it in the wild. Example from Mozilla's public looker-hub repo: + # https://github.com/mozilla/looker-hub/blob/f491ca51ce1add87c338e6723fd49bc6ae4015ca/fenix/explores/activation.explore.lkml#L7 + # As such, we try to handle it but are as defensive as possible. + + non_base_project_name = project_name + if project_name == _BASE_PROJECT_NAME and root_project_name is not None: + non_base_project_name = root_project_name + if non_base_project_name != _BASE_PROJECT_NAME and inc.startswith( + f"/{non_base_project_name}/" + ): + # This might be a local include. Let's make sure that '/{project_name}' doesn't + # exist as normal include in the project. + if not pathlib.Path( + f"{resolved_project_folder}/{non_base_project_name}" + ).exists(): + path_within_project = pathlib.Path(*pathlib.Path(inc).parts[2:]) + glob_expr = f"{resolved_project_folder}/{path_within_project}" + else: + # Need to handle a relative path. + glob_expr = str(pathlib.Path(path).parent / inc) + # "**" matches an arbitrary number of directories in LookML + # we also resolve these paths to absolute paths so we can de-dup effectively later on + included_files = [ + str(p.resolve()) + for p in [ + pathlib.Path(p) + for p in sorted( + glob.glob(glob_expr, recursive=True) + + glob.glob(f"{glob_expr}.lkml", recursive=True) + ) + ] + # We don't want to match directories. The '**' glob can be used to + # recurse into directories. + if p.is_file() + ] + logger.debug( + f"traversal_path={traversal_path}, included_files = {included_files}, seen_so_far: {seen_so_far}" + ) + if "*" not in inc and not included_files: + reporter.report_failure( + title="Error Resolving Include", + message=f"Cannot resolve include {inc}", + context=f"Path: {path}", + ) + elif not included_files: + reporter.report_failure( + title="Error Resolving Include", + message=f"Did not resolve anything for wildcard include {inc}", + context=f"Path: {path}", + ) + # only load files that we haven't seen so far + included_files = [x for x in included_files if x not in seen_so_far] + for included_file in included_files: + # Filter out dashboards - we get those through the looker source. + if ( + included_file.endswith(".dashboard") + or included_file.endswith(".dashboard.lookml") + or included_file.endswith(".dashboard.lkml") + ): + logger.debug( + f"include '{included_file}' is a dashboard, skipping it" + ) + continue + + logger.debug( + f"Will be loading {included_file}, traversed here via {traversal_path}" + ) + try: + parsed = load_lkml(included_file) + seen_so_far.add(included_file) + if "includes" in parsed: # we have more includes to resolve! + resolved.extend( + LookerModel.resolve_includes( + parsed["includes"], + resolved_project_name, + root_project_name, + base_projects_folder, + included_file, + reporter, + seen_so_far, + traversal_path=traversal_path + + "." + + pathlib.Path(included_file).stem, + ) + ) + except Exception as e: + reporter.report_warning( + title="Error Loading Include File", + message="Failed to load included file", + context=f"Include Details: {included_file}", + exc=e, + ) + # continue in this case, as it might be better to load and resolve whatever we can + + resolved.extend( + [ + ProjectInclude(project=resolved_project_name, include=f) + for f in included_files + ] + ) + return resolved + + +@dataclass +class LookerViewFile: + absolute_file_path: str + connection: Optional[LookerConnectionDefinition] + includes: List[str] + resolved_includes: List[ProjectInclude] + views: List[Dict] + raw_file_content: str + + @classmethod + def from_looker_dict( + cls, + absolute_file_path: str, + looker_view_file_dict: dict, + project_name: str, + root_project_name: Optional[str], + base_projects_folder: Dict[str, pathlib.Path], + raw_file_content: str, + reporter: LookMLSourceReport, + ) -> "LookerViewFile": + logger.debug(f"Loading view file at {absolute_file_path}") + includes = looker_view_file_dict.get("includes", []) + resolved_path = str(pathlib.Path(absolute_file_path).resolve()) + seen_so_far = set() + seen_so_far.add(resolved_path) + resolved_includes = LookerModel.resolve_includes( + includes, + project_name, + root_project_name, + base_projects_folder, + absolute_file_path, + reporter, + seen_so_far=seen_so_far, + ) + logger.debug( + f"resolved_includes for {absolute_file_path} is {resolved_includes}" + ) + views = looker_view_file_dict.get("views", []) + + return cls( + absolute_file_path=absolute_file_path, + connection=None, + includes=includes, + resolved_includes=resolved_includes, + views=views, + raw_file_content=raw_file_content, + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py new file mode 100644 index 0000000000000..1b6619b4c4d28 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py @@ -0,0 +1,113 @@ +import logging +import pathlib +from dataclasses import replace +from typing import Any, Dict, Optional + +from datahub.ingestion.source.looker.lkml_patched import load_lkml +from datahub.ingestion.source.looker.looker_config import LookerConnectionDefinition +from datahub.ingestion.source.looker.looker_dataclasses import LookerViewFile +from datahub.ingestion.source.looker.looker_template_language import ( + resolve_liquid_variable_in_view_dict, +) +from datahub.ingestion.source.looker.lookml_config import ( + _EXPLORE_FILE_EXTENSION, + _VIEW_FILE_EXTENSION, + LookMLSourceReport, +) + +logger = logging.getLogger(__name__) + + +class LookerViewFileLoader: + """ + Loads the looker viewfile at a :path and caches the LookerViewFile in memory + This is to avoid reloading the same file off of disk many times during the recursive include resolution process + """ + + def __init__( + self, + root_project_name: Optional[str], + base_projects_folder: Dict[str, pathlib.Path], + reporter: LookMLSourceReport, + liquid_variable: Dict[Any, Any], + ) -> None: + self.viewfile_cache: Dict[str, LookerViewFile] = {} + self._root_project_name = root_project_name + self._base_projects_folder = base_projects_folder + self.reporter = reporter + self.liquid_variable = liquid_variable + + def is_view_seen(self, path: str) -> bool: + return path in self.viewfile_cache + + def _load_viewfile( + self, project_name: str, path: str, reporter: LookMLSourceReport + ) -> Optional[LookerViewFile]: + # always fully resolve paths to simplify de-dup + path = str(pathlib.Path(path).resolve()) + allowed_extensions = [_VIEW_FILE_EXTENSION, _EXPLORE_FILE_EXTENSION] + matched_any_extension = [ + match for match in [path.endswith(x) for x in allowed_extensions] if match + ] + if not matched_any_extension: + # not a view file + logger.debug( + f"Skipping file {path} because it doesn't appear to be a view file. Matched extensions {allowed_extensions}" + ) + return None + + if self.is_view_seen(str(path)): + return self.viewfile_cache[path] + + try: + with open(path) as file: + raw_file_content = file.read() + except Exception as e: + logger.debug(f"An error occurred while reading path {path}", exc_info=True) + self.reporter.report_failure( + path, f"failed to load view file {path} from disk: {e}" + ) + return None + try: + logger.debug(f"Loading viewfile {path}") + + parsed = load_lkml(path) + + resolve_liquid_variable_in_view_dict( + raw_view=parsed, + liquid_variable=self.liquid_variable, + ) + + looker_viewfile = LookerViewFile.from_looker_dict( + absolute_file_path=path, + looker_view_file_dict=parsed, + project_name=project_name, + root_project_name=self._root_project_name, + base_projects_folder=self._base_projects_folder, + raw_file_content=raw_file_content, + reporter=reporter, + ) + logger.debug(f"adding viewfile for path {path} to the cache") + self.viewfile_cache[path] = looker_viewfile + return looker_viewfile + except Exception as e: + logger.debug(f"An error occurred while parsing path {path}", exc_info=True) + self.reporter.report_failure(path, f"failed to load view file {path}: {e}") + return None + + def load_viewfile( + self, + path: str, + project_name: str, + connection: Optional[LookerConnectionDefinition], + reporter: LookMLSourceReport, + ) -> Optional[LookerViewFile]: + viewfile = self._load_viewfile( + project_name=project_name, + path=path, + reporter=reporter, + ) + if viewfile is None: + return None + + return replace(viewfile, connection=connection) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py new file mode 100644 index 0000000000000..35231d273fbba --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py @@ -0,0 +1,104 @@ +from functools import lru_cache +from typing import ClassVar, Optional, TextIO, cast + +from liquid import Environment +from liquid.ast import Node +from liquid.context import Context +from liquid.parse import expect, get_parser +from liquid.stream import TokenStream +from liquid.tag import Tag +from liquid.template import BoundTemplate +from liquid.token import TOKEN_EXPRESSION, TOKEN_LITERAL, TOKEN_TAG, Token + + +class CustomTagException(Exception): + def __init__(self, message): + super().__init__(message) + + +class ConditionNode(Node): + def __init__(self, tok: Token, sql_or_lookml_reference: str, filter_name: str): + self.tok = tok + + self.sql_or_lookml_reference = sql_or_lookml_reference + + self.filter_name = filter_name + + def render_to_output(self, context: Context, buffer: TextIO) -> Optional[bool]: + filter_value: Optional[str] = cast( + str, context.globals.get(self.filter_name) + ) # to silent lint + + if filter_value is None: + raise CustomTagException( + f'filter {self.filter_name} value is not provided for "condition" tag' + ) + + filter_value = filter_value.strip() + + buffer.write(f"{self.sql_or_lookml_reference}='{filter_value}'") + + return True + + +# Define the custom tag +class ConditionTag(Tag): + """ + ConditionTag is the equivalent implementation of looker's custom liquid tag "condition". + Refer doc: https://cloud.google.com/looker/docs/templated-filters#basic_usage + + Refer doc to see how to write liquid custom tag: https://jg-rp.github.io/liquid/guides/custom-tags + + This class render the below tag as order.region='ap-south-1' if order_region is provided in config.liquid_variables + as order_region: 'ap-south-1' + {% condition order_region %} order.region {% endcondition %} + + """ + + TAG_START: ClassVar[str] = "condition" + TAG_END: ClassVar[str] = "endcondition" + name: str = "condition" + + def __init__(self, env: Environment): + super().__init__(env) + self.parser = get_parser(self.env) + + def parse(self, stream: TokenStream) -> Node: + expect(stream, TOKEN_TAG, value=ConditionTag.TAG_START) + + start_token = stream.current + + stream.next_token() + expect(stream, TOKEN_EXPRESSION) + filter_name: str = stream.current.value.strip() + + stream.next_token() + expect(stream, TOKEN_LITERAL) + + sql_or_lookml_reference: str = stream.current.value.strip() + + stream.next_token() + expect(stream, TOKEN_TAG, value=ConditionTag.TAG_END) + + return ConditionNode( + tok=start_token, + sql_or_lookml_reference=sql_or_lookml_reference, + filter_name=filter_name, + ) + + +custom_tags = [ConditionTag] + + +@lru_cache(maxsize=1) +def _create_env() -> Environment: + env: Environment = Environment() + # register tag. One time activity + for custom_tag in custom_tags: + env.add_tag(custom_tag) + return env + + +def create_template(text: str) -> BoundTemplate: + env: Environment = _create_env() + return env.from_string(text) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index c4ba3146031af..d951a6dbe7a62 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -102,7 +102,7 @@ OwnershipTypeClass, SubTypesClass, ) -from datahub.utilities.advanced_thread_executor import BackpressureAwareExecutor +from datahub.utilities.backpressure_aware_executor import BackpressureAwareExecutor logger = logging.getLogger(__name__) @@ -284,8 +284,9 @@ def _get_views_from_fields(self, fields: List[str]) -> List[str]: views.add(view_name) except AssertionError: self.reporter.report_warning( - key=f"chart-field-{field_name}", - reason="The field was not prefixed by a view name. This can happen when the field references another dynamic field.", + title="Failed to Extract View Name from Field", + message="The field was not prefixed by a view name. This can happen when the field references another dynamic field.", + context=f"Field Name: {field_name}", ) continue @@ -370,7 +371,8 @@ def _get_input_fields_from_query( if field is None: continue - # we haven't loaded in metadata about the explore yet, so we need to wait until explores are populated later to fetch this + # we haven't loaded in metadata about the explore yet, so we need to wait until explores are populated + # later to fetch this result.append( InputFieldElement( name=field, view_field=None, model=query.model, explore=query.view @@ -437,6 +439,7 @@ def _get_looker_dashboard_element( # noqa: C901 for exp in explores ], input_fields=input_fields, + owner=None, ) # Dashboard elements can *alternatively* link to an existing look @@ -488,6 +491,7 @@ def _get_looker_dashboard_element( # noqa: C901 if element.look.folder else None ), + owner=self._get_looker_user(element.look.user_id), ) # Failing the above two approaches, pick out details from result_maker @@ -558,6 +562,7 @@ def _get_looker_dashboard_element( # noqa: C901 LookerExplore(model_name=model, name=exp) for exp in explores ], input_fields=input_fields, + owner=None, ) logger.debug(f"Element {element.title}: Unable to parse LookerDashboardElement") @@ -590,16 +595,18 @@ def _get_chart_type( type_str = dashboard_element.type if not type_str: self.reporter.report_warning( - key=f"looker-chart-{dashboard_element.id}", - reason=f"Chart type {type_str} is missing. Setting to None", + title="Unrecognized Chart Type", + message=f"Chart type {type_str} is not recognized. Setting to None", + context=f"Dashboard Id: {dashboard_element.id}", ) return None try: chart_type = type_mapping[type_str] except KeyError: self.reporter.report_warning( - key=f"looker-chart-{dashboard_element.id}", - reason=f"Chart type {type_str} not supported. Setting to None", + title="Unrecognized Chart Type", + message=f"Chart type {type_str} is not recognized. Setting to None", + context=f"Dashboard Id: {dashboard_element.id}", ) chart_type = None @@ -690,6 +697,10 @@ def _make_chart_metadata_events( ownership = self.get_ownership(dashboard) if ownership is not None: chart_snapshot.aspects.append(ownership) + elif dashboard is None and dashboard_element is not None: + ownership = self.get_ownership(dashboard_element) + if ownership is not None: + chart_snapshot.aspects.append(ownership) chart_mce = MetadataChangeEvent(proposedSnapshot=chart_snapshot) @@ -970,10 +981,10 @@ def _make_dashboard_and_chart_mces( yield from dashboard_events def get_ownership( - self, looker_dashboard: LookerDashboard + self, looker_dashboard_look: Union[LookerDashboard, LookerDashboardElement] ) -> Optional[OwnershipClass]: - if looker_dashboard.owner is not None: - owner_urn = looker_dashboard.owner.get_urn( + if looker_dashboard_look.owner is not None: + owner_urn = looker_dashboard_look.owner.get_urn( self.source_config.strip_user_ids_from_email ) if owner_urn is not None: @@ -1172,7 +1183,7 @@ def _input_fields_from_dashboard_element( if relevant_field is not None: view_field_for_reference = relevant_field - if view_field_for_reference is not None: + if view_field_for_reference and view_field_for_reference.name: fields_for_mcp.append( InputFieldClass( schemaFieldUrn=builder.make_schema_field_urn( @@ -1244,8 +1255,9 @@ def process_dashboard( except SDKError: # A looker dashboard could be deleted in between the list and the get self.reporter.report_warning( - dashboard_id, - f"Error occurred while loading dashboard {dashboard_id}. Skipping.", + title="Error Loading Dashboard", + message="Error occurred while attempting to loading dashboard from Looker API. Skipping.", + context=f"Dashboard ID: {dashboard_id}", ) return [], None, dashboard_id, start_time, datetime.datetime.now() @@ -1255,7 +1267,9 @@ def process_dashboard( or dashboard_object.folder.is_personal_descendant ): self.reporter.report_warning( - dashboard_id, "Dropped due to being a personal folder" + title="Dropped Dashboard", + message="Dropped due to being a personal folder", + context=f"Dashboard ID: {dashboard_id}", ) self.reporter.report_dashboards_dropped(dashboard_id) return [], None, dashboard_id, start_time, datetime.datetime.now() @@ -1381,7 +1395,14 @@ def extract_independent_looks(self) -> Iterable[MetadataWorkUnit]: self.reporter.report_stage_start("extract_independent_looks") logger.debug("Extracting looks not part of Dashboard") - look_fields: List[str] = ["id", "title", "description", "query_id", "folder"] + look_fields: List[str] = [ + "id", + "title", + "description", + "query_id", + "folder", + "user_id", + ] query_fields: List[str] = [ "id", "view", @@ -1426,7 +1447,9 @@ def extract_independent_looks(self) -> Iterable[MetadataWorkUnit]: subtitle_text=look.description, look_id=look.id, dashboard_id=None, # As this is independent look - look=LookWithQuery(query=query, folder=look.folder), + look=LookWithQuery( + query=query, folder=look.folder, user_id=look.user_id + ), ), ) @@ -1524,7 +1547,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: ): # Looks like we tried to extract owners and could not find their email addresses. This is likely a permissions issue self.reporter.report_warning( - "api", + "Failed to extract owner emails", "Failed to extract owners emails for any dashboards. Please enable the see_users permission for your Looker API key", ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py new file mode 100644 index 0000000000000..919d9232a18c5 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py @@ -0,0 +1,115 @@ +import logging +import re +from typing import Any, ClassVar, Dict, Set + +from liquid import Undefined +from liquid.exceptions import LiquidSyntaxError + +from datahub.ingestion.source.looker.looker_liquid_tag import ( + CustomTagException, + create_template, +) +from datahub.ingestion.source.looker.str_functions import ( + remove_extra_spaces_and_newlines, +) + +logger = logging.getLogger(__name__) + + +class SpecialVariable: + SPECIAL_VARIABLE_PATTERN: ClassVar[ + str + ] = r"\b\w+(\.\w+)*\._(is_selected|in_query|is_filtered)\b" + liquid_variable: dict + + def __init__(self, liquid_variable): + self.liquid_variable = liquid_variable + + def _create_new_liquid_variables_with_default( + self, + variables: Set[str], + ) -> dict: + new_dict = {**self.liquid_variable} + + for variable in variables: + keys = variable.split( + "." + ) # variable is defined as view._is_selected or view.field_name._is_selected + + current_dict: dict = new_dict + + for key in keys[:-1]: + + if key not in current_dict: + current_dict[key] = {} + + current_dict = current_dict[key] + + if keys[-1] not in current_dict: + current_dict[keys[-1]] = True + + logger.debug("added special variables in liquid_variable dictionary") + + return new_dict + + def liquid_variable_with_default(self, text: str) -> dict: + variables: Set[str] = set( + [ + text[m.start() : m.end()] + for m in re.finditer(SpecialVariable.SPECIAL_VARIABLE_PATTERN, text) + ] + ) + + # if set is empty then no special variables are found. + if not variables: + return self.liquid_variable + + return self._create_new_liquid_variables_with_default(variables=variables) + + +def resolve_liquid_variable(text: str, liquid_variable: Dict[Any, Any]) -> str: + # Set variable value to NULL if not present in liquid_variable dictionary + Undefined.__str__ = lambda instance: "NULL" # type: ignore + try: + # See is there any special boolean variables are there in the text like _in_query, _is_selected, and + # _is_filtered. Refer doc for more information + # https://cloud.google.com/looker/docs/liquid-variable-reference#usage_of_in_query_is_selected_and_is_filtered + # update in liquid_variable with there default values + liquid_variable = SpecialVariable(liquid_variable).liquid_variable_with_default( + text + ) + # Resolve liquid template + return create_template(text).render(liquid_variable) + except LiquidSyntaxError as e: + logger.warning(f"Unsupported liquid template encountered. error [{e.message}]") + # TODO: There are some tag specific to looker and python-liquid library does not understand them. currently + # we are not parsing such liquid template. + # + # See doc: https://cloud.google.com/looker/docs/templated-filters and look for { % condition region %} + # order.region { % endcondition %} + except CustomTagException as e: + logger.warning(e) + logger.debug(e, exc_info=e) + + return text + + +def resolve_liquid_variable_in_view_dict( + raw_view: dict, liquid_variable: Dict[Any, Any] +) -> None: + if "views" not in raw_view: + return + + for view in raw_view["views"]: + if "sql_table_name" in view: + view["sql_table_name"] = resolve_liquid_variable( + text=remove_extra_spaces_and_newlines(view["sql_table_name"]), + liquid_variable=liquid_variable, + ) + + if "derived_table" in view and "sql" in view["derived_table"]: + # In sql we don't need to remove the extra spaces as sql parser takes care of extra spaces and \n + # while generating URN from sql + view["derived_table"]["sql"] = resolve_liquid_variable( + text=view["derived_table"]["sql"], liquid_variable=liquid_variable + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_view_id_cache.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_view_id_cache.py new file mode 100644 index 0000000000000..aa45bb72d1f46 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_view_id_cache.py @@ -0,0 +1,120 @@ +import logging +from typing import Dict, List, Optional + +from datahub.ingestion.source.looker.looker_common import LookerViewId, ViewFieldValue +from datahub.ingestion.source.looker.looker_config import LookerConnectionDefinition +from datahub.ingestion.source.looker.looker_dataclasses import LookerModel +from datahub.ingestion.source.looker.looker_file_loader import LookerViewFileLoader +from datahub.ingestion.source.looker.lookml_config import ( + _BASE_PROJECT_NAME, + NAME, + LookMLSourceReport, +) + +logger = logging.getLogger(__name__) + + +def determine_view_file_path(base_folder_path: str, absolute_file_path: str) -> str: + splits: List[str] = absolute_file_path.split(base_folder_path, 1) + if len(splits) != 2: + logger.debug( + f"base_folder_path({base_folder_path}) and absolute_file_path({absolute_file_path}) not matching" + ) + return ViewFieldValue.NOT_AVAILABLE.value + + file_path: str = splits[1] + logger.debug(f"file_path={file_path}") + + return file_path.strip( + "/" + ) # strip / from path to make it equivalent to source_file attribute of LookerModelExplore API + + +class LookerViewIdCache: + """ + For view to view lineage we require LookerViewId object to form urn in advance for lineage generation. + The case where a view is referencing to another view using derived table can be located in this cache. + + Example: Consider a view registration_monthly_phasing has below SQL + SELECT * + + FROM ${registration_daily_phasing.SQL_TABLE_NAME} + + {% if date_sel._parameter_value == "'Weekly'"%} + WHERE DW_EFF_DT < DATEADD(DAY, (-DAYOFWEEK(current_date()) - 1),current_date()) + {% endif %} + + While generating MCPs for registration_monthly_phasing, the connector can look for view id + of registration_daily_phasing in this cache to generate the lineage between registration_monthly_phasing + and registration_daily_phasing + + This cache can be used for many other use case. + """ + + looker_model: LookerModel + looker_viewfile_loader: LookerViewFileLoader + project_name: str + model_name: str + reporter: LookMLSourceReport + looker_view_id_cache: Dict[ + str, LookerViewId + ] # Map of view-name as key, and LookerViewId instance as value + + def __init__( + self, + project_name: str, + model_name: str, + looker_model: LookerModel, + looker_viewfile_loader: LookerViewFileLoader, + reporter: LookMLSourceReport, + ): + self.project_name = project_name + self.model_name = model_name + self.looker_model = looker_model + self.looker_viewfile_loader = looker_viewfile_loader + self.looker_view_id_cache = {} + self.reporter = reporter + + def get_looker_view_id( + self, + view_name: str, + base_folder_path: str, + connection: Optional[LookerConnectionDefinition] = None, + ) -> Optional[LookerViewId]: + if view_name in self.looker_view_id_cache: + return self.looker_view_id_cache[view_name] + + for include in self.looker_model.resolved_includes: + included_looker_viewfile = self.looker_viewfile_loader.load_viewfile( + path=include.include, + project_name=include.project, + reporter=self.reporter, + connection=connection, + ) + + if included_looker_viewfile is None: + continue + + for view in included_looker_viewfile.views: + if view[NAME] == view_name: + file_path = determine_view_file_path( + base_folder_path, included_looker_viewfile.absolute_file_path + ) + + current_project_name: str = ( + include.project + if include.project != _BASE_PROJECT_NAME + else self.project_name + ) + + looker_view_id: LookerViewId = LookerViewId( + project_name=current_project_name, + model_name=self.model_name, + view_name=view_name, + file_path=file_path, + ) + + self.looker_view_id_cache[view_name] = looker_view_id + return looker_view_id + + return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_concept_context.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_concept_context.py new file mode 100644 index 0000000000000..e528e578dcf9f --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_concept_context.py @@ -0,0 +1,414 @@ +import itertools +import logging +import re +from typing import Any, Dict, List, Optional + +from datahub.ingestion.source.looker.looker_common import ( + ViewFieldValue, + find_view_from_resolved_includes, +) +from datahub.ingestion.source.looker.looker_config import LookerConnectionDefinition +from datahub.ingestion.source.looker.looker_dataclasses import LookerViewFile +from datahub.ingestion.source.looker.looker_file_loader import LookerViewFileLoader +from datahub.ingestion.source.looker.lookml_config import ( + DERIVED_VIEW_PATTERN, + DERIVED_VIEW_SUFFIX, + NAME, + LookMLSourceReport, +) +from datahub.ingestion.source.looker.lookml_refinement import LookerRefinementResolver + +logger = logging.getLogger(__name__) + + +class LookerFieldContext: + raw_field: Dict[Any, Any] + + def __init__(self, raw_field: Dict[Any, Any]): + self.raw_field = raw_field + + def name(self) -> str: + return self.raw_field[NAME] + + def sql(self) -> Optional[str]: + return self.raw_field.get("sql") + + def column_name_in_sql_attribute(self) -> List[str]: + if self.sql() is None: + # If no "sql" is specified, we assume this is referencing an upstream field + # with the same name. This commonly happens for extends and derived tables. + return [self.name()] + + column_names: List[str] = [] + + sql: Optional[str] = self.sql() + + assert sql # to silent lint false positive + + for upstream_field_match in re.finditer(r"\${TABLE}\.[\"]*([\.\w]+)", sql): + matched_field = upstream_field_match.group(1) + # Remove quotes from field names + matched_field = matched_field.replace('"', "").replace("`", "").lower() + column_names.append(matched_field) + + return column_names + + +class LookerViewContext: + """ + There are six patterns to associate the view's fields with dataset + + Pattern1: + view: view_name { + ... measure and dimension definition i.e. fields of a view + } + + In Pattern1 the fields' upstream dataset name is equivalent to view_name and this dataset should be present in + the connection. + + Pattern2: + view: view_name { + sql_table_name: dataset-name + + ... measure and dimension definition i.e. fields of a view + } + + In Pattern2 the fields' upstream dataset name is mentioned in "sql_table_name" attribute and this dataset + should be present in the connection. + + Pattern3: + view: view_name { + sql_table_name: ".SQL_TABLE_NAME" + + ... measure and dimension definition i.e. fields of a view + } + + In Pattern3 the fields' upstream is another view in same looker project. + + Pattern4: + view: view_name { + derived_table: + sql: + ... SQL select query + + ... measure and dimension definition i.e. fields of a view + } + + In Pattern4 the fields' upstream dataset is the output of sql mentioned in derived_table.sql. + + Pattern5: + view: view_name { + derived_table: + explore_source: + ... LookML native query + + ... measure and dimension definition i.e. fields of a view + } + + In Pattern5 the fields' upstream dataset is the output of LookML native query mentioned in + derived_table.explore_source. + + In all patterns the "sql_table_name" or "derived_table" field might present in parent view instead of current + view (see "extends" doc https://cloud.google.com/looker/docs/reference/param-view-extends) + + In all the patterns the common thing is field definition and fields are defined as + # Dimensions + dimension: id { + primary_key: yes + type: number + sql: ${TABLE}.id ;; + } + + # Measures + measure: total_revenue { + type: sum + sql: ${TABLE}.total_revenue ;; + } + + Here "sql" attribute is referring to column present in upstream dataset. + + This sql can be complex sql, see below example + + dimension: profit_in_dollars { type: number sql: ${TABLE}.revenue_in_dollars - ${TABLE}.cost_in_dollars ;; } + Here "profit_in_dollars" has two upstream columns from upstream dataset i.e. revenue_in_dollars and + cost_in_dollars. + + There is one special case of view definition, which is actually not useful but still a valid lookml definition. We + call it pattern 6. Refer below lookml + + view: customer_facts { + derived_table: { + sql: + SELECT + customer_id, + SUM(sale_price) AS lifetime_spend + FROM + order + WHERE + {% if order.region == "ap-south-1" %} + region = "AWS_AP_SOUTH_1" + {% else %} + region = "GCP_SOUTH_1" + {% endif %} + GROUP BY 1 + ;; + } + } + + The customer_facts view is not useful for looker as there is no field definition, but still such view appears in + connector test-cases, and it might be present on customer side + + For all possible options of "sql" attribute please refer looker doc: + https://cloud.google.com/looker/docs/reference/param-field-sql + + """ + + raw_view: Dict + view_file: LookerViewFile + view_connection: LookerConnectionDefinition + view_file_loader: LookerViewFileLoader + looker_refinement_resolver: LookerRefinementResolver + base_folder_path: str + reporter: LookMLSourceReport + + def __init__( + self, + raw_view: Dict, + view_file: LookerViewFile, + view_connection: LookerConnectionDefinition, + view_file_loader: LookerViewFileLoader, + looker_refinement_resolver: LookerRefinementResolver, + base_folder_path: str, + reporter: LookMLSourceReport, + ): + self.raw_view = raw_view + self.view_file = view_file + self.view_connection = view_connection + self.view_file_loader = view_file_loader + self.looker_refinement_resolver = looker_refinement_resolver + self.base_folder_path = base_folder_path + self.reporter = reporter + + def resolve_extends_view_name( + self, + target_view_name: str, + ) -> Optional[dict]: + # The view could live in the same file. + for raw_view in self.view_file.views: + raw_view_name = raw_view["name"] + if raw_view_name == target_view_name: + return self.looker_refinement_resolver.apply_view_refinement(raw_view) + + # Or, it could live in one of the imports. + view = find_view_from_resolved_includes( + connection=self.view_connection, + resolved_includes=self.view_file.resolved_includes, + looker_viewfile_loader=self.view_file_loader, + target_view_name=target_view_name, + reporter=self.reporter, + ) + + if view: + return self.looker_refinement_resolver.apply_view_refinement(view[1]) + else: + logger.warning( + f"failed to resolve view {target_view_name} included from {self.view_file.absolute_file_path}" + ) + return None + + def get_including_extends( + self, + field: str, + ) -> Optional[Any]: + extends = list( + itertools.chain.from_iterable( + self.raw_view.get("extends", self.raw_view.get("extends__all", [])) + ) + ) + + # First, check the current view. + if field in self.raw_view: + return self.raw_view[field] + + # The field might be defined in another view and this view is extending that view, + # so we resolve this field while taking that into account. + # following Looker's precedence rules. + for extend in reversed(extends): + assert extend != self.raw_view[NAME], "a view cannot extend itself" + extend_view = self.resolve_extends_view_name( + extend, + ) + if not extend_view: + raise NameError( + f"failed to resolve extends view {extend} in view {self.raw_view[NAME]} of" + f" file {self.view_file.absolute_file_path}" + ) + if field in extend_view: + return extend_view[field] + + return None + + def _get_sql_table_name_field(self) -> Optional[str]: + return self.get_including_extends(field="sql_table_name") + + def _is_dot_sql_table_name_present(self) -> bool: + sql_table_name: Optional[str] = self._get_sql_table_name_field() + + if sql_table_name is None: + return False + + if DERIVED_VIEW_SUFFIX in sql_table_name.lower(): + return True + + return False + + def sql_table_name(self) -> str: + sql_table_name: Optional[str] = self._get_sql_table_name_field() + # if sql_table_name field is not set then the table name is equal to view-name + if sql_table_name is None: + return self.raw_view[NAME].lower() + + # sql_table_name is in the format "${view-name}.SQL_TABLE_NAME" + # remove extra characters + if self._is_dot_sql_table_name_present(): + sql_table_name = re.sub(DERIVED_VIEW_PATTERN, r"\1", sql_table_name) + + # Some sql_table_name fields contain quotes like: optimizely."group", just remove the quotes + return sql_table_name.replace('"', "").replace("`", "").lower() + + def derived_table(self) -> Dict[Any, Any]: + """ + This function should only be called if is_native_derived_case return true + """ + derived_table = self.get_including_extends(field="derived_table") + + assert derived_table, "derived_table should not be None" + + return derived_table + + def explore_source(self) -> Dict[Any, Any]: + """ + This function should only be called if is_native_derived_case return true + """ + derived_table = self.derived_table() + + assert derived_table.get("explore_source"), "explore_source should not be None" + + return derived_table["explore_source"] + + def sql(self, transformed: bool = True) -> str: + """ + This function should only be called if is_sql_based_derived_case return true + """ + derived_table = self.derived_table() + + # Looker supports sql fragments that omit the SELECT and FROM parts of the query + # Add those in if we detect that it is missing + sql_query: str = derived_table["sql"] + + if transformed: # update the original sql attribute only if transformed is true + if not re.search(r"SELECT\s", sql_query, flags=re.I): + # add a SELECT clause at the beginning + sql_query = f"SELECT {sql_query}" + + if not re.search(r"FROM\s", sql_query, flags=re.I): + # add a FROM clause at the end + sql_query = f"{sql_query} FROM {self.name()}" + # Get the list of tables in the query + + # Drop ${ and } + sql_query = re.sub(DERIVED_VIEW_PATTERN, r"\1", sql_query) + + return sql_query + + def name(self) -> str: + return self.raw_view[NAME] + + def view_file_name(self) -> str: + splits: List[str] = self.view_file.absolute_file_path.split( + self.base_folder_path, 1 + ) + if len(splits) != 2: + logger.debug( + f"base_folder_path({self.base_folder_path}) and absolute_file_path({self.view_file.absolute_file_path})" + f" not matching" + ) + return ViewFieldValue.NOT_AVAILABLE.value + + file_name: str = splits[1] + logger.debug(f"file_path={file_name}") + + return file_name.strip( + "/" + ) # strip / from path to make it equivalent to source_file attribute of LookerModelExplore API + + def _get_list_dict(self, attribute_name: str) -> List[Dict]: + ans: Optional[List[Dict]] = self.raw_view.get(attribute_name) + if ans is not None: + return ans + return [] + + def dimensions(self) -> List[Dict]: + return self._get_list_dict("dimensions") + + def measures(self) -> List[Dict]: + return self._get_list_dict("measures") + + def dimension_groups(self) -> List[Dict]: + return self._get_list_dict("dimension_groups") + + def is_materialized_derived_view(self) -> bool: + for k in self.derived_table(): + if k in ["datagroup_trigger", "sql_trigger_value", "persist_for"]: + return True + + if "materialized_view" in self.derived_table(): + return self.derived_table()["materialized_view"] == "yes" + + return False + + def is_regular_case(self) -> bool: + # regular-case is pattern1 and 2 where upstream table is either view-name or + # table name mentioned in sql_table_name attribute + if ( + self.is_sql_table_name_referring_to_view() + or self.is_sql_based_derived_case() + or self.is_native_derived_case() + ): + return False + + return True + + def is_sql_table_name_referring_to_view(self) -> bool: + # It is pattern3 + return self._is_dot_sql_table_name_present() + + def is_sql_based_derived_case(self) -> bool: + # It is pattern 4 + if "derived_table" in self.raw_view and "sql" in self.raw_view["derived_table"]: + return True + + return False + + def is_native_derived_case(self) -> bool: + # It is pattern 5 + if ( + "derived_table" in self.raw_view + and "explore_source" in self.raw_view["derived_table"] + ): + return True + + return False + + def is_sql_based_derived_view_without_fields_case(self) -> bool: + # Pattern 6 + fields: List[Dict] = [] + + fields.extend(self.dimensions()) + fields.extend(self.measures()) + fields.extend(self.dimension_groups()) + + if self.is_sql_based_derived_case() and len(fields) == 0: + return True + + return False diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py new file mode 100644 index 0000000000000..aa5719547c03e --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py @@ -0,0 +1,235 @@ +import logging +from dataclasses import dataclass, field as dataclass_field +from datetime import timedelta +from typing import Any, Dict, List, Optional, Union + +import pydantic +from pydantic import root_validator, validator +from pydantic.fields import Field + +from datahub.configuration.common import AllowDenyPattern +from datahub.configuration.git import GitInfo +from datahub.configuration.source_common import EnvConfigMixin +from datahub.configuration.validate_field_rename import pydantic_renamed_field +from datahub.ingestion.source.looker.looker_config import LookerCommonConfig +from datahub.ingestion.source.looker.looker_connection import LookerConnectionDefinition +from datahub.ingestion.source.looker.looker_lib_wrapper import ( + LookerAPI, + LookerAPIConfig, + TransportOptionsConfig, +) +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalSourceReport, + StatefulStaleMetadataRemovalConfig, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionConfigBase, +) +from datahub.utilities.lossy_collections import LossyList + +logger = logging.getLogger(__name__) + +NAME: str = "name" + +_BASE_PROJECT_NAME = "__BASE" + +_EXPLORE_FILE_EXTENSION = ".explore.lkml" + +_VIEW_FILE_EXTENSION = ".view.lkml" + +_MODEL_FILE_EXTENSION = ".model.lkml" + +VIEW_LANGUAGE_LOOKML: str = "lookml" + +VIEW_LANGUAGE_SQL: str = "sql" + +DERIVED_VIEW_SUFFIX = r".sql_table_name" + +DERIVED_VIEW_PATTERN: str = r"\$\{([^}]*)\}" + + +@dataclass +class LookMLSourceReport(StaleEntityRemovalSourceReport): + git_clone_latency: Optional[timedelta] = None + models_discovered: int = 0 + models_dropped: List[str] = dataclass_field(default_factory=LossyList) + views_discovered: int = 0 + views_dropped: List[str] = dataclass_field(default_factory=LossyList) + views_dropped_unreachable: List[str] = dataclass_field(default_factory=LossyList) + query_parse_attempts: int = 0 + query_parse_failures: int = 0 + query_parse_failure_views: List[str] = dataclass_field(default_factory=LossyList) + _looker_api: Optional[LookerAPI] = None + + def report_models_scanned(self) -> None: + self.models_discovered += 1 + + def report_views_scanned(self) -> None: + self.views_discovered += 1 + + def report_models_dropped(self, model: str) -> None: + self.models_dropped.append(model) + + def report_views_dropped(self, view: str) -> None: + self.views_dropped.append(view) + + def report_unreachable_view_dropped(self, view: str) -> None: + self.views_dropped_unreachable.append(view) + + def compute_stats(self) -> None: + if self._looker_api: + self.api_stats = self._looker_api.compute_stats() + return super().compute_stats() + + +class LookMLSourceConfig( + LookerCommonConfig, StatefulIngestionConfigBase, EnvConfigMixin +): + git_info: Optional[GitInfo] = Field( + None, + description="Reference to your git location. If present, supplies handy links to your lookml on the dataset " + "entity page.", + ) + _github_info_deprecated = pydantic_renamed_field("github_info", "git_info") + base_folder: Optional[pydantic.DirectoryPath] = Field( + None, + description="Required if not providing github configuration and deploy keys. A pointer to a local directory (" + "accessible to the ingestion system) where the root of the LookML repo has been checked out (" + "typically via a git clone). This is typically the root folder where the `*.model.lkml` and " + "`*.view.lkml` files are stored. e.g. If you have checked out your LookML repo under " + "`/Users/jdoe/workspace/my-lookml-repo`, then set `base_folder` to " + "`/Users/jdoe/workspace/my-lookml-repo`.", + ) + project_dependencies: Dict[str, Union[pydantic.DirectoryPath, GitInfo]] = Field( + {}, + description="A map of project_name to local directory (accessible to the ingestion system) or Git credentials. " + "Every local_dependencies or private remote_dependency listed in the main project's manifest.lkml file should " + "have a corresponding entry here." + "If a deploy key is not provided, the ingestion system will use the same deploy key as the main project. ", + ) + connection_to_platform_map: Optional[Dict[str, LookerConnectionDefinition]] = Field( + None, + description="A mapping of [Looker connection names](" + "https://docs.looker.com/reference/model-params/connection-for-model) to DataHub platform, " + "database, and schema values.", + ) + model_pattern: AllowDenyPattern = Field( + AllowDenyPattern.allow_all(), + description="List of regex patterns for LookML models to include in the extraction.", + ) + view_pattern: AllowDenyPattern = Field( + AllowDenyPattern.allow_all(), + description="List of regex patterns for LookML views to include in the extraction.", + ) + parse_table_names_from_sql: bool = Field(True, description="See note below.") + sql_parser: str = Field( + "datahub.utilities.sql_parser.DefaultSQLParser", description="See note below." + ) + api: Optional[LookerAPIConfig] + project_name: Optional[str] = Field( + None, + description="Required if you don't specify the `api` section. The project name within which all the model " + "files live. See (https://docs.looker.com/data-modeling/getting-started/how-project-works) to " + "understand what the Looker project name should be. The simplest way to see your projects is to " + "click on `Develop` followed by `Manage LookML Projects` in the Looker application.", + ) + transport_options: Optional[TransportOptionsConfig] = Field( + None, + description="Populates the [TransportOptions](https://github.com/looker-open-source/sdk-codegen/blob" + "/94d6047a0d52912ac082eb91616c1e7c379ab262/python/looker_sdk/rtl/transport.py#L70) struct for " + "looker client", + ) + max_file_snippet_length: int = Field( + 512000, # 512KB should be plenty + description="When extracting the view definition from a lookml file, the maximum number of characters to " + "extract.", + ) + emit_reachable_views_only: bool = Field( + True, + description="When enabled, only views that are reachable from explores defined in the model files are emitted", + ) + populate_sql_logic_for_missing_descriptions: bool = Field( + False, + description="When enabled, field descriptions will include the sql logic for computed fields if descriptions " + "are missing", + ) + process_isolation_for_sql_parsing: bool = Field( + False, + description="When enabled, sql parsing will be executed in a separate process to prevent memory leaks.", + ) + stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = Field( + default=None, description="" + ) + process_refinements: bool = Field( + False, + description="When enabled, looker refinement will be processed to adapt an existing view.", + ) + + liquid_variable: Dict[Any, Any] = Field( + {}, + description="A dictionary containing Liquid variables and their corresponding values, utilized in SQL-defined " + "derived views. The Liquid template will be resolved in view.derived_table.sql and " + "view.sql_table_name. Defaults to an empty dictionary.", + ) + + @validator("connection_to_platform_map", pre=True) + def convert_string_to_connection_def(cls, conn_map): + # Previous version of config supported strings in connection map. This upconverts strings to ConnectionMap + for key in conn_map: + if isinstance(conn_map[key], str): + platform = conn_map[key] + if "." in platform: + platform_db_split = conn_map[key].split(".") + connection = LookerConnectionDefinition( + platform=platform_db_split[0], + default_db=platform_db_split[1], + default_schema="", + ) + conn_map[key] = connection + else: + logger.warning( + f"Connection map for {key} provides platform {platform} but does not provide a default " + f"database name. This might result in failed resolution" + ) + conn_map[key] = LookerConnectionDefinition( + platform=platform, default_db="", default_schema="" + ) + return conn_map + + @root_validator(skip_on_failure=True) + def check_either_connection_map_or_connection_provided(cls, values): + """Validate that we must either have a connection map or an api credential""" + if not values.get("connection_to_platform_map", {}) and not values.get( + "api", {} + ): + raise ValueError( + "Neither api not connection_to_platform_map config was found. LookML source requires either api " + "credentials for Looker or a map of connection names to platform identifiers to work correctly" + ) + return values + + @root_validator(skip_on_failure=True) + def check_either_project_name_or_api_provided(cls, values): + """Validate that we must either have a project name or an api credential to fetch project names""" + if not values.get("project_name") and not values.get("api"): + raise ValueError( + "Neither project_name not an API credential was found. LookML source requires either api credentials " + "for Looker or a project_name to accurately name views and models." + ) + return values + + @validator("base_folder", always=True) + def check_base_folder_if_not_provided( + cls, v: Optional[pydantic.DirectoryPath], values: Dict[str, Any] + ) -> Optional[pydantic.DirectoryPath]: + if v is None: + git_info: Optional[GitInfo] = values.get("git_info") + if git_info: + if not git_info.deploy_key: + logger.warning( + "git_info is provided, but no SSH key is present. If the repo is not public, we'll fail to " + "clone it." + ) + else: + raise ValueError("Neither base_folder nor git_info has been provided.") + return v diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_refinement.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_refinement.py new file mode 100644 index 0000000000000..892ed79754a1c --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_refinement.py @@ -0,0 +1,251 @@ +import copy +import itertools +import logging +from typing import ClassVar, Dict, List, Set + +from datahub.ingestion.source.looker.looker_config import LookerConnectionDefinition +from datahub.ingestion.source.looker.looker_dataclasses import LookerModel +from datahub.ingestion.source.looker.looker_view_id_cache import LookerViewFileLoader +from datahub.ingestion.source.looker.lookml_config import ( + NAME, + LookMLSourceConfig, + LookMLSourceReport, +) + +logger = logging.getLogger(__name__) + + +class LookerRefinementResolver: + """ + Refinements are a way to "edit" an existing view or explore. + Refer: https://cloud.google.com/looker/docs/lookml-refinements + + A refinement to an existing view/explore is only applied if it's refinement is reachable from include files in a + model. For refinement applied order please refer: + https://cloud.google.com/looker/docs/lookml-refinements#refinements_are_applied_in_order + """ + + REFINEMENT_PREFIX: ClassVar[str] = "+" + DIMENSIONS: ClassVar[str] = "dimensions" + MEASURES: ClassVar[str] = "measures" + DIMENSION_GROUPS: ClassVar[str] = "dimension_groups" + EXTENDS: ClassVar[str] = "extends" + EXTENDS_ALL: ClassVar[str] = "extends__all" + + looker_model: LookerModel + looker_viewfile_loader: LookerViewFileLoader + connection_definition: LookerConnectionDefinition + source_config: LookMLSourceConfig + reporter: LookMLSourceReport + view_refinement_cache: Dict[ + str, dict + ] # Map of view-name as key, and it is raw view dictionary after applying refinement process + explore_refinement_cache: Dict[ + str, dict + ] # Map of explore-name as key, and it is raw view dictionary after applying refinement process + + def __init__( + self, + looker_model: LookerModel, + looker_viewfile_loader: LookerViewFileLoader, + connection_definition: LookerConnectionDefinition, + source_config: LookMLSourceConfig, + reporter: LookMLSourceReport, + ): + self.looker_model = looker_model + self.looker_viewfile_loader = looker_viewfile_loader + self.connection_definition = connection_definition + self.source_config = source_config + self.reporter = reporter + self.view_refinement_cache = {} + self.explore_refinement_cache = {} + + @staticmethod + def is_refinement(view_name: str) -> bool: + return view_name.startswith(LookerRefinementResolver.REFINEMENT_PREFIX) + + @staticmethod + def merge_column( + original_dict: dict, refinement_dict: dict, key: str + ) -> List[dict]: + """ + Merge a dimension/measure/other column with one from a refinement. + This follows the process documented at https://help.looker.com/hc/en-us/articles/4419773929107-LookML-refinements + """ + merge_column: List[dict] = [] + original_value: List[dict] = original_dict.get(key, []) + refine_value: List[dict] = refinement_dict.get(key, []) + # name is required field, not going to be None + original_column_map = {column[NAME]: column for column in original_value} + refine_column_map = {column[NAME]: column for column in refine_value} + for existing_column_name in original_column_map: + existing_column = original_column_map[existing_column_name] + refine_column = refine_column_map.get(existing_column_name) + if refine_column is not None: + existing_column.update(refine_column) + + merge_column.append(existing_column) + + # merge any remaining column from refine_column_map + for new_column_name in refine_column_map: + if new_column_name not in original_column_map: + merge_column.append(refine_column_map[new_column_name]) + + return merge_column + + @staticmethod + def merge_and_set_column( + new_raw_view: dict, refinement_view: dict, key: str + ) -> None: + merged_column = LookerRefinementResolver.merge_column( + new_raw_view, refinement_view, key + ) + if merged_column: + new_raw_view[key] = merged_column + + @staticmethod + def merge_refinements(raw_view: dict, refinement_views: List[dict]) -> dict: + """ + Iterate over refinement_views and merge parameter of each view with raw_view. + Detail of merging order can be found at https://cloud.google.com/looker/docs/lookml-refinements + """ + new_raw_view: dict = copy.deepcopy(raw_view) + + for refinement_view in refinement_views: + # Merge dimension and measure + # TODO: low priority: handle additive parameters + # https://cloud.google.com/looker/docs/lookml-refinements#some_parameters_are_additive + + # Merge Dimension + LookerRefinementResolver.merge_and_set_column( + new_raw_view, refinement_view, LookerRefinementResolver.DIMENSIONS + ) + # Merge Measure + LookerRefinementResolver.merge_and_set_column( + new_raw_view, refinement_view, LookerRefinementResolver.MEASURES + ) + # Merge Dimension Group + LookerRefinementResolver.merge_and_set_column( + new_raw_view, refinement_view, LookerRefinementResolver.DIMENSION_GROUPS + ) + + return new_raw_view + + def get_refinements(self, views: List[dict], view_name: str) -> List[dict]: + """ + Refinement syntax for view and explore are same. + This function can be used to filter out view/explore refinement from raw dictionary list + """ + view_refinement_name: str = self.REFINEMENT_PREFIX + view_name + refined_views: List[dict] = [] + + for raw_view in views: + if view_refinement_name == raw_view[NAME]: + refined_views.append(raw_view) + + return refined_views + + def get_refinement_from_model_includes(self, view_name: str) -> List[dict]: + refined_views: List[dict] = [] + + for include in self.looker_model.resolved_includes: + included_looker_viewfile = self.looker_viewfile_loader.load_viewfile( + include.include, + include.project, + self.connection_definition, + self.reporter, + ) + + if not included_looker_viewfile: + continue + + refined_views.extend( + self.get_refinements(included_looker_viewfile.views, view_name) + ) + + return refined_views + + def should_skip_processing(self, raw_view_name: str) -> bool: + if LookerRefinementResolver.is_refinement(raw_view_name): + return True + + if self.source_config.process_refinements is False: + return True + + return False + + def apply_view_refinement(self, raw_view: dict) -> dict: + """ + Looker process the lkml file in include order and merge the all refinement to original view. + """ + assert raw_view.get(NAME) is not None + + raw_view_name: str = raw_view[NAME] + + if self.should_skip_processing(raw_view_name): + return raw_view + + if raw_view_name in self.view_refinement_cache: + logger.debug(f"Returning applied refined view {raw_view_name} from cache") + return self.view_refinement_cache[raw_view_name] + + logger.debug(f"Processing refinement for view {raw_view_name}") + + refinement_views: List[dict] = self.get_refinement_from_model_includes( + raw_view_name + ) + + self.view_refinement_cache[raw_view_name] = self.merge_refinements( + raw_view, refinement_views + ) + + return self.view_refinement_cache[raw_view_name] + + @staticmethod + def add_extended_explore( + raw_explore: dict, refinement_explores: List[Dict] + ) -> None: + extended_explores: Set[str] = set() + for view in refinement_explores: + extends = list( + itertools.chain.from_iterable( + view.get( + LookerRefinementResolver.EXTENDS, + view.get(LookerRefinementResolver.EXTENDS_ALL, []), + ) + ) + ) + extended_explores.update(extends) + + if extended_explores: # if it is not empty then add to the original view + raw_explore[LookerRefinementResolver.EXTENDS] = list(extended_explores) + + def apply_explore_refinement(self, raw_view: dict) -> dict: + """ + In explore refinement `extends` parameter is additive. + Refer looker refinement document: https://cloud.google.com/looker/docs/lookml-refinements#additive + """ + assert raw_view.get(NAME) is not None + + raw_view_name: str = raw_view[NAME] + + if self.should_skip_processing(raw_view_name): + return raw_view + + if raw_view_name in self.explore_refinement_cache: + logger.debug( + f"Returning applied refined explore {raw_view_name} from cache" + ) + return self.explore_refinement_cache[raw_view_name] + + logger.debug(f"Processing refinement for explore {raw_view_name}") + + refinement_explore: List[dict] = self.get_refinements( + self.looker_model.explores, raw_view_name + ) + + self.add_extended_explore(raw_view, refinement_explore) + + self.explore_refinement_cache[raw_view_name] = raw_view + + return self.explore_refinement_cache[raw_view_name] diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py index 4a872f8b1a025..223d168dbe033 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py @@ -1,39 +1,16 @@ -import copy -import glob -import itertools import logging import pathlib -import re import tempfile -from dataclasses import dataclass, field as dataclass_field, replace -from datetime import datetime, timedelta, timezone -from typing import ( - Any, - ClassVar, - Dict, - Iterable, - List, - Optional, - Set, - Tuple, - Type, - Union, -) +from collections import OrderedDict +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Dict, Iterable, List, Optional, Set, Tuple import lkml import lkml.simple -import pydantic from looker_sdk.error import SDKError -from looker_sdk.sdk.api40.models import DBConnection -from pydantic import root_validator, validator -from pydantic.fields import Field -import datahub.emitter.mce_builder as builder -from datahub.configuration import ConfigModel -from datahub.configuration.common import AllowDenyPattern, ConfigurationError from datahub.configuration.git import GitInfo -from datahub.configuration.source_common import EnvConfigMixin -from datahub.configuration.validate_field_rename import pydantic_renamed_field from datahub.emitter.mce_builder import make_schema_field_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.mcp_builder import gen_containers @@ -45,7 +22,6 @@ platform_name, support_status, ) -from datahub.ingestion.api.registry import import_path from datahub.ingestion.api.source import MetadataWorkUnitProcessor, SourceCapability from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.common.subtypes import ( @@ -56,28 +32,46 @@ from datahub.ingestion.source.looker.lkml_patched import load_lkml from datahub.ingestion.source.looker.looker_common import ( CORPUSER_DATAHUB, - LookerCommonConfig, LookerExplore, LookerUtil, LookerViewId, - ProjectInclude, ViewField, ViewFieldType, ViewFieldValue, + deduplicate_fields, gen_project_key, ) -from datahub.ingestion.source.looker.looker_lib_wrapper import ( - LookerAPI, - LookerAPIConfig, - TransportOptionsConfig, +from datahub.ingestion.source.looker.looker_connection import ( + get_connection_def_based_on_connection_string, +) +from datahub.ingestion.source.looker.looker_lib_wrapper import LookerAPI +from datahub.ingestion.source.looker.looker_view_id_cache import ( + LookerModel, + LookerViewFileLoader, + LookerViewIdCache, +) +from datahub.ingestion.source.looker.lookml_concept_context import ( + LookerFieldContext, + LookerViewContext, +) +from datahub.ingestion.source.looker.lookml_config import ( + _BASE_PROJECT_NAME, + _MODEL_FILE_EXTENSION, + VIEW_LANGUAGE_LOOKML, + VIEW_LANGUAGE_SQL, + LookerConnectionDefinition, + LookMLSourceConfig, + LookMLSourceReport, +) +from datahub.ingestion.source.looker.lookml_refinement import LookerRefinementResolver +from datahub.ingestion.source.looker.view_upstream import ( + AbstractViewUpstream, + create_view_upstream, ) from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalHandler, - StaleEntityRemovalSourceReport, - StatefulStaleMetadataRemovalConfig, ) from datahub.ingestion.source.state.stateful_ingestion_base import ( - StatefulIngestionConfigBase, StatefulIngestionSourceBase, ) from datahub.metadata.com.linkedin.pegasus2avro.common import BrowsePaths, Status @@ -100,1000 +94,21 @@ FineGrainedLineageUpstreamTypeClass, SubTypesClass, ) -from datahub.utilities.lossy_collections import LossyList -from datahub.utilities.sql_parser import SQLParser +from datahub.sql_parsing.sqlglot_lineage import ColumnRef logger = logging.getLogger(__name__) -_BASE_PROJECT_NAME = "__BASE" - -_EXPLORE_FILE_EXTENSION = ".explore.lkml" -_VIEW_FILE_EXTENSION = ".view.lkml" -_MODEL_FILE_EXTENSION = ".model.lkml" - - -def deduplicate_fields(fields: List[ViewField]) -> List[ViewField]: - # Remove duplicates filed from self.fields - # Logic is: If more than a field has same ViewField.name then keep only one filed where ViewField.field_type - # is DIMENSION_GROUP. - # Looker Constraint: - # - Any field declared as dimension or measure can be redefined as dimension_group. - # - Any field declared in dimension can't be redefined in measure and vice-versa. - - dimension_group_field_names: List[str] = [ - field.name - for field in fields - if field.field_type == ViewFieldType.DIMENSION_GROUP - ] - - new_fields: List[ViewField] = [] - - for field in fields: - if ( - field.name in dimension_group_field_names - and field.field_type != ViewFieldType.DIMENSION_GROUP - ): - continue - - new_fields.append(field) - - return new_fields - - -def _get_bigquery_definition( - looker_connection: DBConnection, -) -> Tuple[str, Optional[str], Optional[str]]: - platform = "bigquery" - # bigquery project ids are returned in the host field - db = looker_connection.host - schema = looker_connection.database - return (platform, db, schema) - - -def _get_generic_definition( - looker_connection: DBConnection, platform: Optional[str] = None -) -> Tuple[str, Optional[str], Optional[str]]: - if platform is None: - # We extract the platform from the dialect name - dialect_name = looker_connection.dialect_name - assert dialect_name is not None - # generally the first part of the dialect name before _ is the name of the platform - # versions are encoded as numbers and can be removed - # e.g. spark1 or hive2 or druid_18 - platform = re.sub(r"[0-9]+", "", dialect_name.split("_")[0]) - - assert ( - platform is not None - ), f"Failed to extract a valid platform from connection {looker_connection}" - db = looker_connection.database - schema = looker_connection.schema # ok for this to be None - return (platform, db, schema) - - -class LookerConnectionDefinition(ConfigModel): - platform: str - default_db: str - default_schema: Optional[str] # Optional since some sources are two-level only - platform_instance: Optional[str] = None - platform_env: Optional[str] = Field( - default=None, - description="The environment that the platform is located in. Leaving this empty will inherit defaults from the top level Looker configuration", - ) - - @validator("platform_env") - def platform_env_must_be_one_of(cls, v: Optional[str]) -> Optional[str]: - if v is not None: - return EnvConfigMixin.env_must_be_one_of(v) - return v - - @validator("platform", "default_db", "default_schema") - def lower_everything(cls, v): - """We lower case all strings passed in to avoid casing issues later""" - if v is not None: - return v.lower() - - @classmethod - def from_looker_connection( - cls, looker_connection: DBConnection - ) -> "LookerConnectionDefinition": - """Dialect definitions are here: https://docs.looker.com/setup-and-management/database-config""" - extractors: Dict[str, Any] = { - "^bigquery": _get_bigquery_definition, - ".*": _get_generic_definition, - } - - if looker_connection.dialect_name is None: - raise ConfigurationError( - f"Unable to fetch a fully filled out connection for {looker_connection.name}. Please check your API permissions." - ) - for extractor_pattern, extracting_function in extractors.items(): - if re.match(extractor_pattern, looker_connection.dialect_name): - (platform, db, schema) = extracting_function(looker_connection) - return cls(platform=platform, default_db=db, default_schema=schema) - raise ConfigurationError( - f"Could not find an appropriate platform for looker_connection: {looker_connection.name} with dialect: {looker_connection.dialect_name}" - ) - - -class LookMLSourceConfig( - LookerCommonConfig, StatefulIngestionConfigBase, EnvConfigMixin -): - git_info: Optional[GitInfo] = Field( - None, - description="Reference to your git location. If present, supplies handy links to your lookml on the dataset entity page.", - ) - _github_info_deprecated = pydantic_renamed_field("github_info", "git_info") - base_folder: Optional[pydantic.DirectoryPath] = Field( - None, - description="Required if not providing github configuration and deploy keys. A pointer to a local directory (accessible to the ingestion system) where the root of the LookML repo has been checked out (typically via a git clone). This is typically the root folder where the `*.model.lkml` and `*.view.lkml` files are stored. e.g. If you have checked out your LookML repo under `/Users/jdoe/workspace/my-lookml-repo`, then set `base_folder` to `/Users/jdoe/workspace/my-lookml-repo`.", - ) - project_dependencies: Dict[str, Union[pydantic.DirectoryPath, GitInfo]] = Field( - {}, - description="A map of project_name to local directory (accessible to the ingestion system) or Git credentials. " - "Every local_dependencies or private remote_dependency listed in the main project's manifest.lkml file should have a corresponding entry here. " - "If a deploy key is not provided, the ingestion system will use the same deploy key as the main project. ", - ) - connection_to_platform_map: Optional[Dict[str, LookerConnectionDefinition]] = Field( - None, - description="A mapping of [Looker connection names](https://docs.looker.com/reference/model-params/connection-for-model) to DataHub platform, database, and schema values.", - ) - model_pattern: AllowDenyPattern = Field( - AllowDenyPattern.allow_all(), - description="List of regex patterns for LookML models to include in the extraction.", - ) - view_pattern: AllowDenyPattern = Field( - AllowDenyPattern.allow_all(), - description="List of regex patterns for LookML views to include in the extraction.", - ) - parse_table_names_from_sql: bool = Field(False, description="See note below.") - sql_parser: str = Field( - "datahub.utilities.sql_parser.DefaultSQLParser", description="See note below." - ) - api: Optional[LookerAPIConfig] - project_name: Optional[str] = Field( - None, - description="Required if you don't specify the `api` section. The project name within which all the model files live. See (https://docs.looker.com/data-modeling/getting-started/how-project-works) to understand what the Looker project name should be. The simplest way to see your projects is to click on `Develop` followed by `Manage LookML Projects` in the Looker application.", - ) - transport_options: Optional[TransportOptionsConfig] = Field( - None, - description="Populates the [TransportOptions](https://github.com/looker-open-source/sdk-codegen/blob/94d6047a0d52912ac082eb91616c1e7c379ab262/python/looker_sdk/rtl/transport.py#L70) struct for looker client", - ) - max_file_snippet_length: int = Field( - 512000, # 512KB should be plenty - description="When extracting the view definition from a lookml file, the maximum number of characters to extract.", - ) - emit_reachable_views_only: bool = Field( - True, - description="When enabled, only views that are reachable from explores defined in the model files are emitted", - ) - populate_sql_logic_for_missing_descriptions: bool = Field( - False, - description="When enabled, field descriptions will include the sql logic for computed fields if descriptions are missing", - ) - process_isolation_for_sql_parsing: bool = Field( - False, - description="When enabled, sql parsing will be executed in a separate process to prevent memory leaks.", - ) - stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = Field( - default=None, description="" - ) - process_refinements: bool = Field( - False, - description="When enabled, looker refinement will be processed to adapt an existing view.", - ) - - @validator("connection_to_platform_map", pre=True) - def convert_string_to_connection_def(cls, conn_map): - # Previous version of config supported strings in connection map. This upconverts strings to ConnectionMap - for key in conn_map: - if isinstance(conn_map[key], str): - platform = conn_map[key] - if "." in platform: - platform_db_split = conn_map[key].split(".") - connection = LookerConnectionDefinition( - platform=platform_db_split[0], - default_db=platform_db_split[1], - default_schema="", - ) - conn_map[key] = connection - else: - logger.warning( - f"Connection map for {key} provides platform {platform} but does not provide a default database name. This might result in failed resolution" - ) - conn_map[key] = LookerConnectionDefinition( - platform=platform, default_db="", default_schema="" - ) - return conn_map - - @root_validator(skip_on_failure=True) - def check_either_connection_map_or_connection_provided(cls, values): - """Validate that we must either have a connection map or an api credential""" - if not values.get("connection_to_platform_map", {}) and not values.get( - "api", {} - ): - raise ValueError( - "Neither api not connection_to_platform_map config was found. LookML source requires either api credentials for Looker or a map of connection names to platform identifiers to work correctly" - ) - return values - - @root_validator(skip_on_failure=True) - def check_either_project_name_or_api_provided(cls, values): - """Validate that we must either have a project name or an api credential to fetch project names""" - if not values.get("project_name") and not values.get("api"): - raise ValueError( - "Neither project_name not an API credential was found. LookML source requires either api credentials for Looker or a project_name to accurately name views and models." - ) - return values - - @validator("base_folder", always=True) - def check_base_folder_if_not_provided( - cls, v: Optional[pydantic.DirectoryPath], values: Dict[str, Any] - ) -> Optional[pydantic.DirectoryPath]: - if v is None: - git_info: Optional[GitInfo] = values.get("git_info") - if git_info: - if not git_info.deploy_key: - logger.warning( - "git_info is provided, but no SSH key is present. If the repo is not public, we'll fail to clone it." - ) - else: - raise ValueError("Neither base_folder nor git_info has been provided.") - return v - - -@dataclass -class LookMLSourceReport(StaleEntityRemovalSourceReport): - git_clone_latency: Optional[timedelta] = None - models_discovered: int = 0 - models_dropped: List[str] = dataclass_field(default_factory=LossyList) - views_discovered: int = 0 - views_dropped: List[str] = dataclass_field(default_factory=LossyList) - views_dropped_unreachable: List[str] = dataclass_field(default_factory=LossyList) - query_parse_attempts: int = 0 - query_parse_failures: int = 0 - query_parse_failure_views: List[str] = dataclass_field(default_factory=LossyList) - _looker_api: Optional[LookerAPI] = None - - def report_models_scanned(self) -> None: - self.models_discovered += 1 - - def report_views_scanned(self) -> None: - self.views_discovered += 1 - - def report_models_dropped(self, model: str) -> None: - self.models_dropped.append(model) - - def report_views_dropped(self, view: str) -> None: - self.views_dropped.append(view) - - def report_unreachable_view_dropped(self, view: str) -> None: - self.views_dropped_unreachable.append(view) - - def compute_stats(self) -> None: - if self._looker_api: - self.api_stats = self._looker_api.compute_stats() - return super().compute_stats() - - -@dataclass -class LookerModel: - connection: str - includes: List[str] - explores: List[dict] - resolved_includes: List[ProjectInclude] - - @staticmethod - def from_looker_dict( - looker_model_dict: dict, - base_project_name: str, - root_project_name: Optional[str], - base_projects_folders: Dict[str, pathlib.Path], - path: str, - reporter: LookMLSourceReport, - ) -> "LookerModel": - logger.debug(f"Loading model from {path}") - connection = looker_model_dict["connection"] - includes = looker_model_dict.get("includes", []) - resolved_includes = LookerModel.resolve_includes( - includes, - base_project_name, - root_project_name, - base_projects_folders, - path, - reporter, - seen_so_far=set(), - traversal_path=pathlib.Path(path).stem, - ) - logger.debug(f"{path} has resolved_includes: {resolved_includes}") - explores = looker_model_dict.get("explores", []) - - explore_files = [ - x.include - for x in resolved_includes - if x.include.endswith(_EXPLORE_FILE_EXTENSION) - ] - for included_file in explore_files: - try: - parsed = load_lkml(included_file) - included_explores = parsed.get("explores", []) - explores.extend(included_explores) - except Exception as e: - reporter.report_warning( - path, f"Failed to load {included_file} due to {e}" - ) - # continue in this case, as it might be better to load and resolve whatever we can - - return LookerModel( - connection=connection, - includes=includes, - resolved_includes=resolved_includes, - explores=explores, - ) - - @staticmethod - def resolve_includes( - includes: List[str], - project_name: str, - root_project_name: Optional[str], - base_projects_folder: Dict[str, pathlib.Path], - path: str, - reporter: LookMLSourceReport, - seen_so_far: Set[str], - traversal_path: str = "", # a cosmetic parameter to aid debugging - ) -> List[ProjectInclude]: - """Resolve ``include`` statements in LookML model files to a list of ``.lkml`` files. - - For rules on how LookML ``include`` statements are written, see - https://docs.looker.com/data-modeling/getting-started/ide-folders#wildcard_examples - """ - - resolved = [] - for inc in includes: - # Filter out dashboards - we get those through the looker source. - if ( - inc.endswith(".dashboard") - or inc.endswith(".dashboard.lookml") - or inc.endswith(".dashboard.lkml") - ): - logger.debug(f"include '{inc}' is a dashboard, skipping it") - continue - - resolved_project_name = project_name - resolved_project_folder = str(base_projects_folder[project_name]) - - # Massage the looker include into a valid glob wildcard expression - if inc.startswith("//"): - # remote include, let's see if we have the project checked out locally - (remote_project, project_local_path) = inc[2:].split("/", maxsplit=1) - if remote_project in base_projects_folder: - resolved_project_folder = str(base_projects_folder[remote_project]) - glob_expr = f"{resolved_project_folder}/{project_local_path}" - resolved_project_name = remote_project - else: - logger.warning( - f"Resolving {inc} failed. Could not find a locally checked out reference for {remote_project}" - ) - continue - elif inc.startswith("/"): - glob_expr = f"{resolved_project_folder}{inc}" - - # The include path is sometimes '/{project_name}/{path_within_project}' - # instead of '//{project_name}/{path_within_project}' or '/{path_within_project}'. - # - # TODO: I can't seem to find any documentation on this pattern, but we definitely - # have seen it in the wild. Example from Mozilla's public looker-hub repo: - # https://github.com/mozilla/looker-hub/blob/f491ca51ce1add87c338e6723fd49bc6ae4015ca/fenix/explores/activation.explore.lkml#L7 - # As such, we try to handle it but are as defensive as possible. - - non_base_project_name = project_name - if project_name == _BASE_PROJECT_NAME and root_project_name is not None: - non_base_project_name = root_project_name - if non_base_project_name != _BASE_PROJECT_NAME and inc.startswith( - f"/{non_base_project_name}/" - ): - # This might be a local include. Let's make sure that '/{project_name}' doesn't - # exist as normal include in the project. - if not pathlib.Path( - f"{resolved_project_folder}/{non_base_project_name}" - ).exists(): - path_within_project = pathlib.Path(*pathlib.Path(inc).parts[2:]) - glob_expr = f"{resolved_project_folder}/{path_within_project}" - else: - # Need to handle a relative path. - glob_expr = str(pathlib.Path(path).parent / inc) - # "**" matches an arbitrary number of directories in LookML - # we also resolve these paths to absolute paths so we can de-dup effectively later on - included_files = [ - str(p.resolve()) - for p in [ - pathlib.Path(p) - for p in sorted( - glob.glob(glob_expr, recursive=True) - + glob.glob(f"{glob_expr}.lkml", recursive=True) - ) - ] - # We don't want to match directories. The '**' glob can be used to - # recurse into directories. - if p.is_file() - ] - logger.debug( - f"traversal_path={traversal_path}, included_files = {included_files}, seen_so_far: {seen_so_far}" - ) - if "*" not in inc and not included_files: - reporter.report_failure(path, f"cannot resolve include {inc}") - elif not included_files: - reporter.report_failure( - path, f"did not resolve anything for wildcard include {inc}" - ) - # only load files that we haven't seen so far - included_files = [x for x in included_files if x not in seen_so_far] - for included_file in included_files: - # Filter out dashboards - we get those through the looker source. - if ( - included_file.endswith(".dashboard") - or included_file.endswith(".dashboard.lookml") - or included_file.endswith(".dashboard.lkml") - ): - logger.debug( - f"include '{included_file}' is a dashboard, skipping it" - ) - continue - - logger.debug( - f"Will be loading {included_file}, traversed here via {traversal_path}" - ) - try: - parsed = load_lkml(included_file) - seen_so_far.add(included_file) - if "includes" in parsed: # we have more includes to resolve! - resolved.extend( - LookerModel.resolve_includes( - parsed["includes"], - resolved_project_name, - root_project_name, - base_projects_folder, - included_file, - reporter, - seen_so_far, - traversal_path=traversal_path - + "." - + pathlib.Path(included_file).stem, - ) - ) - except Exception as e: - reporter.report_warning( - path, f"Failed to load {included_file} due to {e}" - ) - # continue in this case, as it might be better to load and resolve whatever we can - - resolved.extend( - [ - ProjectInclude(project=resolved_project_name, include=f) - for f in included_files - ] - ) - return resolved - - -@dataclass -class LookerViewFile: - absolute_file_path: str - connection: Optional[LookerConnectionDefinition] - includes: List[str] - resolved_includes: List[ProjectInclude] - views: List[Dict] - raw_file_content: str - - @classmethod - def from_looker_dict( - cls, - absolute_file_path: str, - looker_view_file_dict: dict, - project_name: str, - root_project_name: Optional[str], - base_projects_folder: Dict[str, pathlib.Path], - raw_file_content: str, - reporter: LookMLSourceReport, - ) -> "LookerViewFile": - logger.debug(f"Loading view file at {absolute_file_path}") - includes = looker_view_file_dict.get("includes", []) - resolved_path = str(pathlib.Path(absolute_file_path).resolve()) - seen_so_far = set() - seen_so_far.add(resolved_path) - resolved_includes = LookerModel.resolve_includes( - includes, - project_name, - root_project_name, - base_projects_folder, - absolute_file_path, - reporter, - seen_so_far=seen_so_far, - ) - logger.debug( - f"resolved_includes for {absolute_file_path} is {resolved_includes}" - ) - views = looker_view_file_dict.get("views", []) - - return cls( - absolute_file_path=absolute_file_path, - connection=None, - includes=includes, - resolved_includes=resolved_includes, - views=views, - raw_file_content=raw_file_content, - ) - - -@dataclass -class SQLInfo: - table_names: List[str] - column_names: List[str] - - -class LookerViewFileLoader: - """ - Loads the looker viewfile at a :path and caches the LookerViewFile in memory - This is to avoid reloading the same file off of disk many times during the recursive include resolution process - """ - - def __init__( - self, - root_project_name: Optional[str], - base_projects_folder: Dict[str, pathlib.Path], - reporter: LookMLSourceReport, - ) -> None: - self.viewfile_cache: Dict[str, LookerViewFile] = {} - self._root_project_name = root_project_name - self._base_projects_folder = base_projects_folder - self.reporter = reporter - - def is_view_seen(self, path: str) -> bool: - return path in self.viewfile_cache - - def _load_viewfile( - self, project_name: str, path: str, reporter: LookMLSourceReport - ) -> Optional[LookerViewFile]: - # always fully resolve paths to simplify de-dup - path = str(pathlib.Path(path).resolve()) - allowed_extensions = [_VIEW_FILE_EXTENSION, _EXPLORE_FILE_EXTENSION] - matched_any_extension = [ - match for match in [path.endswith(x) for x in allowed_extensions] if match - ] - if not matched_any_extension: - # not a view file - logger.debug( - f"Skipping file {path} because it doesn't appear to be a view file. Matched extensions {allowed_extensions}" - ) - return None - - if self.is_view_seen(str(path)): - return self.viewfile_cache[path] - - try: - with open(path) as file: - raw_file_content = file.read() - except Exception as e: - self.reporter.report_failure(path, f"failed to load view file: {e}") - return None - try: - logger.debug(f"Loading viewfile {path}") - parsed = load_lkml(path) - looker_viewfile = LookerViewFile.from_looker_dict( - absolute_file_path=path, - looker_view_file_dict=parsed, - project_name=project_name, - root_project_name=self._root_project_name, - base_projects_folder=self._base_projects_folder, - raw_file_content=raw_file_content, - reporter=reporter, - ) - logger.debug(f"adding viewfile for path {path} to the cache") - self.viewfile_cache[path] = looker_viewfile - return looker_viewfile - except Exception as e: - self.reporter.report_failure(path, f"failed to load view file: {e}") - return None - - def load_viewfile( - self, - path: str, - project_name: str, - connection: Optional[LookerConnectionDefinition], - reporter: LookMLSourceReport, - ) -> Optional[LookerViewFile]: - viewfile = self._load_viewfile( - project_name=project_name, path=path, reporter=reporter - ) - if viewfile is None: - return None - - return replace(viewfile, connection=connection) - - -class LookerRefinementResolver: - """ - Refinements are a way to "edit" an existing view or explore. - Refer: https://cloud.google.com/looker/docs/lookml-refinements - - A refinement to an existing view/explore is only applied if it's refinement is reachable from include files in a model. - For refinement applied order please refer: https://cloud.google.com/looker/docs/lookml-refinements#refinements_are_applied_in_order - """ - - REFINEMENT_PREFIX: ClassVar[str] = "+" - DIMENSIONS: ClassVar[str] = "dimensions" - MEASURES: ClassVar[str] = "measures" - DIMENSION_GROUPS: ClassVar[str] = "dimension_groups" - NAME: ClassVar[str] = "name" - EXTENDS: ClassVar[str] = "extends" - EXTENDS_ALL: ClassVar[str] = "extends__all" - - looker_model: LookerModel - looker_viewfile_loader: LookerViewFileLoader - connection_definition: LookerConnectionDefinition - source_config: LookMLSourceConfig - reporter: LookMLSourceReport - view_refinement_cache: Dict[ - str, dict - ] # Map of view-name as key, and it is raw view dictionary after applying refinement process - explore_refinement_cache: Dict[ - str, dict - ] # Map of explore-name as key, and it is raw view dictionary after applying refinement process - - def __init__( - self, - looker_model: LookerModel, - looker_viewfile_loader: LookerViewFileLoader, - connection_definition: LookerConnectionDefinition, - source_config: LookMLSourceConfig, - reporter: LookMLSourceReport, - ): - self.looker_model = looker_model - self.looker_viewfile_loader = looker_viewfile_loader - self.connection_definition = connection_definition - self.source_config = source_config - self.reporter = reporter - self.view_refinement_cache = {} - self.explore_refinement_cache = {} - - @staticmethod - def is_refinement(view_name: str) -> bool: - return view_name.startswith(LookerRefinementResolver.REFINEMENT_PREFIX) - - @staticmethod - def merge_column( - original_dict: dict, refinement_dict: dict, key: str - ) -> List[dict]: - """ - Merge a dimension/measure/other column with one from a refinement. - This follows the process documented at https://help.looker.com/hc/en-us/articles/4419773929107-LookML-refinements - """ - merge_column: List[dict] = [] - original_value: List[dict] = original_dict.get(key, []) - refine_value: List[dict] = refinement_dict.get(key, []) - # name is required field, not going to be None - original_column_map = { - column[LookerRefinementResolver.NAME]: column for column in original_value - } - refine_column_map = { - column[LookerRefinementResolver.NAME]: column for column in refine_value - } - for existing_column_name in original_column_map: - existing_column = original_column_map[existing_column_name] - refine_column = refine_column_map.get(existing_column_name) - if refine_column is not None: - existing_column.update(refine_column) - - merge_column.append(existing_column) - - # merge any remaining column from refine_column_map - for new_column_name in refine_column_map: - if new_column_name not in original_column_map: - merge_column.append(refine_column_map[new_column_name]) - - return merge_column - - @staticmethod - def merge_and_set_column( - new_raw_view: dict, refinement_view: dict, key: str - ) -> None: - merged_column = LookerRefinementResolver.merge_column( - new_raw_view, refinement_view, key - ) - if merged_column: - new_raw_view[key] = merged_column - - @staticmethod - def merge_refinements(raw_view: dict, refinement_views: List[dict]) -> dict: - """ - Iterate over refinement_views and merge parameter of each view with raw_view. - Detail of merging order can be found at https://cloud.google.com/looker/docs/lookml-refinements - """ - new_raw_view: dict = copy.deepcopy(raw_view) - - for refinement_view in refinement_views: - # Merge dimension and measure - # TODO: low priority: handle additive parameters - # https://cloud.google.com/looker/docs/lookml-refinements#some_parameters_are_additive - - # Merge Dimension - LookerRefinementResolver.merge_and_set_column( - new_raw_view, refinement_view, LookerRefinementResolver.DIMENSIONS - ) - # Merge Measure - LookerRefinementResolver.merge_and_set_column( - new_raw_view, refinement_view, LookerRefinementResolver.MEASURES - ) - # Merge Dimension Group - LookerRefinementResolver.merge_and_set_column( - new_raw_view, refinement_view, LookerRefinementResolver.DIMENSION_GROUPS - ) - - return new_raw_view - - def get_refinements(self, views: List[dict], view_name: str) -> List[dict]: - """ - Refinement syntax for view and explore are same. - This function can be used to filter out view/explore refinement from raw dictionary list - """ - view_refinement_name: str = self.REFINEMENT_PREFIX + view_name - refined_views: List[dict] = [] - - for raw_view in views: - if view_refinement_name == raw_view[LookerRefinementResolver.NAME]: - refined_views.append(raw_view) - - return refined_views - - def get_refinement_from_model_includes(self, view_name: str) -> List[dict]: - refined_views: List[dict] = [] - - for include in self.looker_model.resolved_includes: - included_looker_viewfile = self.looker_viewfile_loader.load_viewfile( - include.include, - include.project, - self.connection_definition, - self.reporter, - ) - - if not included_looker_viewfile: - continue - - refined_views.extend( - self.get_refinements(included_looker_viewfile.views, view_name) - ) - - return refined_views - - def should_skip_processing(self, raw_view_name: str) -> bool: - if LookerRefinementResolver.is_refinement(raw_view_name): - return True - - if self.source_config.process_refinements is False: - return True - - return False - - def apply_view_refinement(self, raw_view: dict) -> dict: - """ - Looker process the lkml file in include order and merge the all refinement to original view. - """ - assert raw_view.get(LookerRefinementResolver.NAME) is not None - - raw_view_name: str = raw_view[LookerRefinementResolver.NAME] - - if self.should_skip_processing(raw_view_name): - return raw_view - - if raw_view_name in self.view_refinement_cache: - logger.debug(f"Returning applied refined view {raw_view_name} from cache") - return self.view_refinement_cache[raw_view_name] - - logger.debug(f"Processing refinement for view {raw_view_name}") - - refinement_views: List[dict] = self.get_refinement_from_model_includes( - raw_view_name - ) - - self.view_refinement_cache[raw_view_name] = self.merge_refinements( - raw_view, refinement_views - ) - - return self.view_refinement_cache[raw_view_name] - - @staticmethod - def add_extended_explore( - raw_explore: dict, refinement_explores: List[Dict] - ) -> None: - extended_explores: Set[str] = set() - for view in refinement_explores: - extends = list( - itertools.chain.from_iterable( - view.get( - LookerRefinementResolver.EXTENDS, - view.get(LookerRefinementResolver.EXTENDS_ALL, []), - ) - ) - ) - extended_explores.update(extends) - - if extended_explores: # if it is not empty then add to the original view - raw_explore[LookerRefinementResolver.EXTENDS] = list(extended_explores) - - def apply_explore_refinement(self, raw_view: dict) -> dict: - """ - In explore refinement `extends` parameter is additive. - Refer looker refinement document: https://cloud.google.com/looker/docs/lookml-refinements#additive - """ - assert raw_view.get(LookerRefinementResolver.NAME) is not None - - raw_view_name: str = raw_view[LookerRefinementResolver.NAME] - - if self.should_skip_processing(raw_view_name): - return raw_view - - if raw_view_name in self.explore_refinement_cache: - logger.debug( - f"Returning applied refined explore {raw_view_name} from cache" - ) - return self.explore_refinement_cache[raw_view_name] - - logger.debug(f"Processing refinement for explore {raw_view_name}") - - refinement_explore: List[dict] = self.get_refinements( - self.looker_model.explores, raw_view_name - ) - - self.add_extended_explore(raw_view, refinement_explore) - - self.explore_refinement_cache[raw_view_name] = raw_view - - return self.explore_refinement_cache[raw_view_name] - - -VIEW_LANGUAGE_LOOKML: str = "lookml" -VIEW_LANGUAGE_SQL: str = "sql" - - -def _find_view_from_resolved_includes( - connection: Optional[LookerConnectionDefinition], - resolved_includes: List[ProjectInclude], - looker_viewfile_loader: LookerViewFileLoader, - target_view_name: str, - reporter: LookMLSourceReport, -) -> Optional[Tuple[ProjectInclude, dict]]: - # It could live in one of the included files. We do not know which file the base view - # lives in, so we try them all! - for include in resolved_includes: - included_looker_viewfile = looker_viewfile_loader.load_viewfile( - include.include, include.project, connection, reporter - ) - if not included_looker_viewfile: - continue - for raw_view in included_looker_viewfile.views: - raw_view_name = raw_view["name"] - # Make sure to skip loading view we are currently trying to resolve - if raw_view_name == target_view_name: - return include, raw_view - - return None - - -_SQL_FUNCTIONS = ["UNNEST"] - @dataclass class LookerView: id: LookerViewId absolute_file_path: str connection: LookerConnectionDefinition - sql_table_names: List[str] - upstream_explores: List[str] + upstream_dataset_urns: List[str] fields: List[ViewField] raw_file_content: str view_details: Optional[ViewProperties] = None - @classmethod - def _import_sql_parser_cls(cls, sql_parser_path: str) -> Type[SQLParser]: - assert "." in sql_parser_path, "sql_parser-path must contain a ." - parser_cls = import_path(sql_parser_path) - - if not issubclass(parser_cls, SQLParser): - raise ValueError(f"must be derived from {SQLParser}; got {parser_cls}") - return parser_cls - - @classmethod - def _get_sql_info( - cls, sql: str, sql_parser_path: str, use_external_process: bool = True - ) -> SQLInfo: - parser_cls = cls._import_sql_parser_cls(sql_parser_path) - - try: - parser_instance: SQLParser = parser_cls( - sql, use_external_process=use_external_process - ) - except Exception as e: - logger.warning(f"Sql parser failed on {sql} with {e}") - return SQLInfo(table_names=[], column_names=[]) - - sql_table_names: List[str] - try: - sql_table_names = parser_instance.get_tables() - except Exception as e: - logger.warning(f"Sql parser failed on {sql} with {e}") - sql_table_names = [] - - try: - column_names: List[str] = parser_instance.get_columns() - except Exception as e: - logger.warning(f"Sql parser failed on {sql} with {e}") - column_names = [] - - logger.debug(f"Column names parsed = {column_names}") - # Drop table names with # in them - sql_table_names = [t for t in sql_table_names if "#" not in t] - - # Remove quotes from table names - sql_table_names = [t.replace('"', "") for t in sql_table_names] - sql_table_names = [t.replace("`", "") for t in sql_table_names] - # Remove reserved words from table names - sql_table_names = [ - t for t in sql_table_names if t.upper() not in _SQL_FUNCTIONS - ] - - return SQLInfo(table_names=sql_table_names, column_names=column_names) - - @classmethod - def _get_fields( - cls, - field_list: List[Dict], - type_cls: ViewFieldType, - extract_column_level_lineage: bool, - populate_sql_logic_in_descriptions: bool, - ) -> List[ViewField]: - fields = [] - for field_dict in field_list: - is_primary_key = field_dict.get("primary_key", "no") == "yes" - name = field_dict["name"] - native_type = field_dict.get("type", "string") - default_description = ( - f"sql:{field_dict['sql']}" - if "sql" in field_dict and populate_sql_logic_in_descriptions - else "" - ) - - description = field_dict.get("description", default_description) - label = field_dict.get("label", "") - upstream_fields = [] - if extract_column_level_lineage: - if field_dict.get("sql") is not None: - for upstream_field_match in re.finditer( - r"\${TABLE}\.[\"]*([\.\w]+)", field_dict["sql"] - ): - matched_field = upstream_field_match.group(1) - # Remove quotes from field names - matched_field = ( - matched_field.replace('"', "").replace("`", "").lower() - ) - upstream_fields.append(matched_field) - else: - # If no SQL is specified, we assume this is referencing an upstream field - # with the same name. This commonly happens for extends and derived tables. - upstream_fields.append(name) - - upstream_fields = sorted(list(set(upstream_fields))) - - field = ViewField( - name=name, - type=native_type, - label=label, - description=description, - is_primary_key=is_primary_key, - field_type=type_cls, - upstream_fields=upstream_fields, - ) - fields.append(field) - return fields - @classmethod def determine_view_file_path( cls, base_folder_path: str, absolute_file_path: str @@ -1116,352 +131,114 @@ def determine_view_file_path( def from_looker_dict( cls, project_name: str, - base_folder_path: str, model_name: str, - looker_view: dict, - connection: LookerConnectionDefinition, - looker_viewfile: LookerViewFile, - looker_viewfile_loader: LookerViewFileLoader, - looker_refinement_resolver: LookerRefinementResolver, + view_context: LookerViewContext, + looker_view_id_cache: LookerViewIdCache, reporter: LookMLSourceReport, max_file_snippet_length: int, - parse_table_names_from_sql: bool = False, - sql_parser_path: str = "datahub.utilities.sql_parser.DefaultSQLParser", + config: LookMLSourceConfig, + ctx: PipelineContext, extract_col_level_lineage: bool = False, populate_sql_logic_in_descriptions: bool = False, - process_isolation_for_sql_parsing: bool = False, ) -> Optional["LookerView"]: - view_name = looker_view["name"] + + view_name = view_context.name() + logger.debug(f"Handling view {view_name} in model {model_name}") - # The sql_table_name might be defined in another view and this view is extending that view, - # so we resolve this field while taking that into account. - sql_table_name: Optional[str] = LookerView.get_including_extends( + + looker_view_id: LookerViewId = LookerViewId( + project_name=project_name, + model_name=model_name, view_name=view_name, - looker_view=looker_view, - connection=connection, - looker_viewfile=looker_viewfile, - looker_viewfile_loader=looker_viewfile_loader, - looker_refinement_resolver=looker_refinement_resolver, - field="sql_table_name", - reporter=reporter, + file_path=view_context.view_file_name(), ) - # Some sql_table_name fields contain quotes like: optimizely."group", just remove the quotes - sql_table_name = ( - sql_table_name.replace('"', "").replace("`", "") - if sql_table_name is not None - else None - ) - derived_table = LookerView.get_including_extends( - view_name=view_name, - looker_view=looker_view, - connection=connection, - looker_viewfile=looker_viewfile, - looker_viewfile_loader=looker_viewfile_loader, - looker_refinement_resolver=looker_refinement_resolver, - field="derived_table", + view_upstream: AbstractViewUpstream = create_view_upstream( + view_context=view_context, + looker_view_id_cache=looker_view_id_cache, + config=config, + ctx=ctx, reporter=reporter, ) - dimensions = cls._get_fields( - looker_view.get("dimensions", []), - ViewFieldType.DIMENSION, - extract_col_level_lineage, - populate_sql_logic_in_descriptions=populate_sql_logic_in_descriptions, - ) - dimension_groups = cls._get_fields( - looker_view.get("dimension_groups", []), - ViewFieldType.DIMENSION_GROUP, - extract_col_level_lineage, - populate_sql_logic_in_descriptions=populate_sql_logic_in_descriptions, - ) - measures = cls._get_fields( - looker_view.get("measures", []), - ViewFieldType.MEASURE, - extract_col_level_lineage, - populate_sql_logic_in_descriptions=populate_sql_logic_in_descriptions, - ) - fields: List[ViewField] = dimensions + dimension_groups + measures + field_type_vs_raw_fields = OrderedDict( + { + ViewFieldType.DIMENSION: view_context.dimensions(), + ViewFieldType.DIMENSION_GROUP: view_context.dimension_groups(), + ViewFieldType.MEASURE: view_context.measures(), + } + ) # in order to maintain order in golden file - fields = deduplicate_fields(fields) + view_fields: List[ViewField] = [] - # Prep "default" values for the view, which will be overridden by the logic below. - view_logic = looker_viewfile.raw_file_content[:max_file_snippet_length] - sql_table_names: List[str] = [] - upstream_explores: List[str] = [] - - if derived_table is not None: - # Derived tables can either be a SQL query or a LookML explore. - # See https://cloud.google.com/looker/docs/derived-tables. - - if "sql" in derived_table: - view_logic = derived_table["sql"] - view_lang = VIEW_LANGUAGE_SQL - - # Parse SQL to extract dependencies. - if parse_table_names_from_sql: - ( - fields, - sql_table_names, - ) = cls._extract_metadata_from_derived_table_sql( - reporter, - sql_parser_path, - view_name, - sql_table_name, - view_logic, - fields, - use_external_process=process_isolation_for_sql_parsing, + for field_type, fields in field_type_vs_raw_fields.items(): + for field in fields: + upstream_column_ref: List[ColumnRef] = [] + if extract_col_level_lineage: + upstream_column_ref = view_upstream.get_upstream_column_ref( + field_context=LookerFieldContext(raw_field=field) ) - elif "explore_source" in derived_table: - # This is called a "native derived table". - # See https://cloud.google.com/looker/docs/creating-ndts. - explore_source = derived_table["explore_source"] - - # We want this to render the full lkml block - # e.g. explore_source: source_name { ... } - # As such, we use the full derived_table instead of the explore_source. - view_logic = str(lkml.dump(derived_table))[:max_file_snippet_length] - view_lang = VIEW_LANGUAGE_LOOKML - - ( - fields, - upstream_explores, - ) = cls._extract_metadata_from_derived_table_explore( - reporter, view_name, explore_source, fields + view_fields.append( + ViewField.view_fields_from_dict( + field_dict=field, + upstream_column_ref=upstream_column_ref, + type_cls=field_type, + populate_sql_logic_in_descriptions=populate_sql_logic_in_descriptions, + ) ) - materialized = False - for k in derived_table: - if k in ["datagroup_trigger", "sql_trigger_value", "persist_for"]: - materialized = True - if "materialized_view" in derived_table: - materialized = derived_table["materialized_view"] == "yes" + # special case where view is defined as derived sql, however fields are not defined + if ( + len(view_fields) == 0 + and view_context.is_sql_based_derived_view_without_fields_case() + ): + view_fields = view_upstream.create_fields() + + view_fields = deduplicate_fields(view_fields) + + # Prep "default" values for the view, which will be overridden by the logic below. + view_logic = view_context.view_file.raw_file_content[:max_file_snippet_length] + + if view_context.is_sql_based_derived_case(): + view_logic = view_context.sql(transformed=False) + # Parse SQL to extract dependencies. + view_details = ViewProperties( + materialized=False, + viewLogic=view_logic, + viewLanguage=VIEW_LANGUAGE_SQL, + ) + elif view_context.is_native_derived_case(): + # We want this to render the full lkml block + # e.g. explore_source: source_name { ... } + # As such, we use the full derived_table instead of the explore_source. + view_logic = str(lkml.dump(view_context.derived_table()))[ + :max_file_snippet_length + ] + view_lang = VIEW_LANGUAGE_LOOKML + + materialized = view_context.is_materialized_derived_view() view_details = ViewProperties( materialized=materialized, viewLogic=view_logic, viewLanguage=view_lang ) else: - # If not a derived table, then this view essentially wraps an existing - # object in the database. If sql_table_name is set, there is a single - # dependency in the view, on the sql_table_name. - # Otherwise, default to the view name as per the docs: - # https://docs.looker.com/reference/view-params/sql_table_name-for-view - sql_table_names = ( - [view_name] if sql_table_name is None else [sql_table_name] - ) view_details = ViewProperties( materialized=False, viewLogic=view_logic, viewLanguage=VIEW_LANGUAGE_LOOKML, ) - file_path = LookerView.determine_view_file_path( - base_folder_path, looker_viewfile.absolute_file_path - ) - return LookerView( - id=LookerViewId( - project_name=project_name, - model_name=model_name, - view_name=view_name, - file_path=file_path, - ), - absolute_file_path=looker_viewfile.absolute_file_path, - connection=connection, - sql_table_names=sql_table_names, - upstream_explores=upstream_explores, - fields=fields, - raw_file_content=looker_viewfile.raw_file_content, + id=looker_view_id, + absolute_file_path=view_context.view_file.absolute_file_path, + connection=view_context.view_connection, + upstream_dataset_urns=view_upstream.get_upstream_dataset_urn(), + fields=view_fields, + raw_file_content=view_context.view_file.raw_file_content, view_details=view_details, ) - @classmethod - def _extract_metadata_from_derived_table_sql( - cls, - reporter: LookMLSourceReport, - sql_parser_path: str, - view_name: str, - sql_table_name: Optional[str], - sql_query: str, - fields: List[ViewField], - use_external_process: bool, - ) -> Tuple[List[ViewField], List[str]]: - sql_table_names: List[str] = [] - - logger.debug(f"Parsing sql from derived table section of view: {view_name}") - reporter.query_parse_attempts += 1 - - # Skip queries that contain liquid variables. We currently don't parse them correctly. - # Docs: https://cloud.google.com/looker/docs/liquid-variable-reference. - # TODO: also support ${EXTENDS} and ${TABLE} - if "{%" in sql_query: - try: - # test if parsing works - sql_info: SQLInfo = cls._get_sql_info( - sql_query, sql_parser_path, use_external_process - ) - if not sql_info.table_names: - raise Exception("Failed to find any tables") - except Exception: - logger.debug( - f"{view_name}: SQL Parsing didn't return any tables, trying a hail-mary" - ) - # A hail-mary simple parse. - for maybe_table_match in re.finditer( - r"FROM\s*([a-zA-Z0-9_.`]+)", sql_query - ): - if maybe_table_match.group(1) not in sql_table_names: - sql_table_names.append(maybe_table_match.group(1)) - return fields, sql_table_names - - # Looker supports sql fragments that omit the SELECT and FROM parts of the query - # Add those in if we detect that it is missing - if not re.search(r"SELECT\s", sql_query, flags=re.I): - # add a SELECT clause at the beginning - sql_query = f"SELECT {sql_query}" - if not re.search(r"FROM\s", sql_query, flags=re.I): - # add a FROM clause at the end - sql_query = f"{sql_query} FROM {sql_table_name if sql_table_name is not None else view_name}" - # Get the list of tables in the query - try: - sql_info = cls._get_sql_info( - sql_query, sql_parser_path, use_external_process - ) - sql_table_names = sql_info.table_names - column_names = sql_info.column_names - - if not fields: - # it seems like the view is defined purely as sql, let's try using the column names to populate the schema - fields = [ - # set types to unknown for now as our sql parser doesn't give us column types yet - ViewField(c, "", "unknown", "", ViewFieldType.UNKNOWN) - for c in sorted(column_names) - ] - # remove fields or sql tables that contain liquid variables - fields = [f for f in fields if "{%" not in f.name] - - if not sql_info.table_names: - reporter.query_parse_failures += 1 - reporter.query_parse_failure_views.append(view_name) - except Exception as e: - reporter.query_parse_failures += 1 - reporter.report_warning( - f"looker-view-{view_name}", - f"Failed to parse sql query, lineage will not be accurate. Exception: {e}", - ) - - sql_table_names = [table for table in sql_table_names if "{%" not in table] - - return fields, sql_table_names - - @classmethod - def _extract_metadata_from_derived_table_explore( - cls, - reporter: LookMLSourceReport, - view_name: str, - explore_source: dict, - fields: List[ViewField], - ) -> Tuple[List[ViewField], List[str]]: - logger.debug( - f"Parsing explore_source from derived table section of view: {view_name}" - ) - - upstream_explores = [explore_source["name"]] - - explore_columns = explore_source.get("columns", []) - # TODO: We currently don't support column-level lineage for derived_column. - # In order to support it, we'd need to parse the `sql` field of the derived_column. - - # The fields in the view are actually references to the fields in the explore. - # As such, we need to perform an extra mapping step to update - # the upstream column names. - for field in fields: - for i, upstream_field in enumerate(field.upstream_fields): - # Find the matching column in the explore. - for explore_column in explore_columns: - if explore_column["name"] == upstream_field: - field.upstream_fields[i] = explore_column.get( - "field", explore_column["name"] - ) - break - - return fields, upstream_explores - - @classmethod - def resolve_extends_view_name( - cls, - connection: LookerConnectionDefinition, - looker_viewfile: LookerViewFile, - looker_viewfile_loader: LookerViewFileLoader, - looker_refinement_resolver: LookerRefinementResolver, - target_view_name: str, - reporter: LookMLSourceReport, - ) -> Optional[dict]: - # The view could live in the same file. - for raw_view in looker_viewfile.views: - raw_view_name = raw_view["name"] - if raw_view_name == target_view_name: - return looker_refinement_resolver.apply_view_refinement(raw_view) - - # Or, it could live in one of the imports. - view = _find_view_from_resolved_includes( - connection, - looker_viewfile.resolved_includes, - looker_viewfile_loader, - target_view_name, - reporter, - ) - if view: - return looker_refinement_resolver.apply_view_refinement(view[1]) - else: - logger.warning( - f"failed to resolve view {target_view_name} included from {looker_viewfile.absolute_file_path}" - ) - return None - - @classmethod - def get_including_extends( - cls, - view_name: str, - looker_view: dict, - connection: LookerConnectionDefinition, - looker_viewfile: LookerViewFile, - looker_viewfile_loader: LookerViewFileLoader, - looker_refinement_resolver: LookerRefinementResolver, - field: str, - reporter: LookMLSourceReport, - ) -> Optional[Any]: - extends = list( - itertools.chain.from_iterable( - looker_view.get("extends", looker_view.get("extends__all", [])) - ) - ) - - # First, check the current view. - if field in looker_view: - return looker_view[field] - - # Then, check the views this extends, following Looker's precedence rules. - for extend in reversed(extends): - assert extend != view_name, "a view cannot extend itself" - extend_view = LookerView.resolve_extends_view_name( - connection, - looker_viewfile, - looker_viewfile_loader, - looker_refinement_resolver, - extend, - reporter, - ) - if not extend_view: - raise NameError( - f"failed to resolve extends view {extend} in view {view_name} of file {looker_viewfile.absolute_file_path}" - ) - if field in extend_view: - return extend_view[field] - - return None - @dataclass class LookerRemoteDependency: @@ -1516,6 +293,7 @@ class LookMLSource(StatefulIngestionSourceBase): def __init__(self, config: LookMLSourceConfig, ctx: PipelineContext): super().__init__(config, ctx) self.source_config = config + self.ctx = ctx self.reporter = LookMLSourceReport() # To keep track of projects (containers) which have already been ingested @@ -1526,10 +304,11 @@ def __init__(self, config: LookMLSourceConfig, ctx: PipelineContext): self.reporter._looker_api = self.looker_client try: self.looker_client.all_connections() - except SDKError: + except SDKError as err: raise ValueError( - "Failed to retrieve connections from looker client. Please check to ensure that you have manage_models permission enabled on this API key." - ) + "Failed to retrieve connections from looker client. Please check to ensure that you have " + "manage_models permission enabled on this API key." + ) from err def _load_model(self, path: str) -> LookerModel: logger.debug(f"Loading model from file {path}") @@ -1544,142 +323,14 @@ def _load_model(self, path: str) -> LookerModel: ) return looker_model - def _platform_names_have_2_parts(self, platform: str) -> bool: - return platform in {"hive", "mysql", "athena"} - - def _generate_fully_qualified_name( - self, sql_table_name: str, connection_def: LookerConnectionDefinition - ) -> str: - """Returns a fully qualified dataset name, resolved through a connection definition. - Input sql_table_name can be in three forms: table, db.table, db.schema.table""" - # TODO: This function should be extracted out into a Platform specific naming class since name translations are required across all connectors - - # Bigquery has "project.db.table" which can be mapped to db.schema.table form - # All other relational db's follow "db.schema.table" - # With the exception of mysql, hive, athena which are "db.table" - - # first detect which one we have - parts = len(sql_table_name.split(".")) - - if parts == 3: - # fully qualified, but if platform is of 2-part, we drop the first level - if self._platform_names_have_2_parts(connection_def.platform): - sql_table_name = ".".join(sql_table_name.split(".")[1:]) - return sql_table_name.lower() - - if parts == 1: - # Bare table form - if self._platform_names_have_2_parts(connection_def.platform): - dataset_name = f"{connection_def.default_db}.{sql_table_name}" - else: - dataset_name = f"{connection_def.default_db}.{connection_def.default_schema}.{sql_table_name}" - return dataset_name.lower() - - if parts == 2: - # if this is a 2 part platform, we are fine - if self._platform_names_have_2_parts(connection_def.platform): - return sql_table_name.lower() - # otherwise we attach the default top-level container - dataset_name = f"{connection_def.default_db}.{sql_table_name}" - return dataset_name.lower() - - self.reporter.report_warning( - key=sql_table_name, reason=f"{sql_table_name} has more than 3 parts." - ) - return sql_table_name.lower() - - def _construct_datalineage_urn( - self, sql_table_name: str, looker_view: LookerView - ) -> str: - logger.debug(f"sql_table_name={sql_table_name}") - connection_def: LookerConnectionDefinition = looker_view.connection - - # Check if table name matches cascading derived tables pattern - # derived tables can be referred to using aliases that look like table_name.SQL_TABLE_NAME - # See https://docs.looker.com/data-modeling/learning-lookml/derived-tables#syntax_for_referencing_a_derived_table - if re.fullmatch(r"\w+\.SQL_TABLE_NAME", sql_table_name, flags=re.I): - sql_table_name = sql_table_name.lower().split(".")[0] - # upstream dataset is a looker view based on current view id's project and model - view_id = LookerViewId( - project_name=looker_view.id.project_name, - model_name=looker_view.id.model_name, - view_name=sql_table_name, - file_path=looker_view.id.file_path, - ) - return view_id.get_urn(self.source_config) - - # Ensure sql_table_name is in canonical form (add in db, schema names) - sql_table_name = self._generate_fully_qualified_name( - sql_table_name, connection_def - ) - - return builder.make_dataset_urn_with_platform_instance( - platform=connection_def.platform, - name=sql_table_name.lower(), - platform_instance=connection_def.platform_instance, - env=connection_def.platform_env or self.source_config.env, - ) - - def _get_connection_def_based_on_connection_string( - self, connection: str - ) -> Optional[LookerConnectionDefinition]: - if self.source_config.connection_to_platform_map is None: - self.source_config.connection_to_platform_map = {} - assert self.source_config.connection_to_platform_map is not None - if connection in self.source_config.connection_to_platform_map: - return self.source_config.connection_to_platform_map[connection] - elif self.looker_client: - try: - looker_connection: DBConnection = self.looker_client.connection( - connection - ) - except SDKError: - logger.error( - f"Failed to retrieve connection {connection} from Looker. This usually happens when the credentials provided are not admin credentials." - ) - else: - try: - connection_def: LookerConnectionDefinition = ( - LookerConnectionDefinition.from_looker_connection( - looker_connection - ) - ) - - # Populate the cache (using the config map) to avoid calling looker again for this connection - self.source_config.connection_to_platform_map[ - connection - ] = connection_def - return connection_def - except ConfigurationError: - self.reporter.report_warning( - f"connection-{connection}", - "Failed to load connection from Looker", - ) - - return None - def _get_upstream_lineage( self, looker_view: LookerView ) -> Optional[UpstreamLineage]: - # Merge dataset upstreams with sql table upstreams. - upstream_dataset_urns = [] - for upstream_explore in looker_view.upstream_explores: - # We're creating a "LookerExplore" just to use the urn generator. - upstream_dataset_urn = LookerExplore( - name=upstream_explore, model_name=looker_view.id.model_name - ).get_explore_urn(self.source_config) - upstream_dataset_urns.append(upstream_dataset_urn) - for sql_table_name in looker_view.sql_table_names: - sql_table_name = sql_table_name.replace('"', "").replace("`", "") - upstream_dataset_urn = self._construct_datalineage_urn( - sql_table_name, looker_view - ) - upstream_dataset_urns.append(upstream_dataset_urn) + upstream_dataset_urns = looker_view.upstream_dataset_urns # Generate the upstream + fine grained lineage objects. upstreams = [] observed_lineage_ts = datetime.now(tz=timezone.utc) - fine_grained_lineages: List[FineGrainedLineageClass] = [] for upstream_dataset_urn in upstream_dataset_urns: upstream = UpstreamClass( dataset=upstream_dataset_urn, @@ -1691,32 +342,27 @@ def _get_upstream_lineage( ) upstreams.append(upstream) - if self.source_config.extract_column_level_lineage and ( - looker_view.view_details is not None - and looker_view.view_details.viewLanguage - != VIEW_LANGUAGE_SQL # we currently only map col-level lineage for views without sql - ): - for field in looker_view.fields: - if field.upstream_fields: - fine_grained_lineage = FineGrainedLineageClass( - upstreamType=FineGrainedLineageUpstreamTypeClass.FIELD_SET, - upstreams=[ - make_schema_field_urn( - upstream_dataset_urn, upstream_field - ) - for upstream_field in field.upstream_fields - ], - downstreamType=FineGrainedLineageDownstreamType.FIELD, - downstreams=[ - make_schema_field_urn( - looker_view.id.get_urn(self.source_config), - field.name, - ) - ], + fine_grained_lineages: List[FineGrainedLineageClass] = [] + + for field in looker_view.fields: + fine_grained_lineages.append( + FineGrainedLineageClass( + upstreamType=FineGrainedLineageUpstreamTypeClass.FIELD_SET, + upstreams=[ + make_schema_field_urn(cll_ref.table, cll_ref.column) + for cll_ref in field.upstream_fields + ], + downstreamType=FineGrainedLineageDownstreamType.FIELD, + downstreams=[ + make_schema_field_urn( + looker_view.id.get_urn(self.source_config), + field.name, ) - fine_grained_lineages.append(fine_grained_lineage) + ], + ) + ) - if upstreams != []: + if upstreams: return UpstreamLineage( upstreams=upstreams, fineGrainedLineages=fine_grained_lineages or None ) @@ -1775,7 +421,6 @@ def _build_dataset_mcps( ) events = [subTypeEvent] if looker_view.view_details is not None: - viewEvent = MetadataChangeProposalWrapper( entityUrn=view_urn, aspect=looker_view.view_details, @@ -1844,7 +489,8 @@ def get_project_name(self, model_name: str) -> str: return model.project_name except SDKError: raise ValueError( - f"Could not locate a project name for model {model_name}. Consider configuring a static project name in your config file" + f"Could not locate a project name for model {model_name}. Consider configuring a static project name " + f"in your config file" ) def get_manifest_if_present(self, folder: pathlib.Path) -> Optional[LookerManifest]: @@ -1930,7 +576,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: if not self.report.events_produced and not self.report.failures: # Don't pass if we didn't produce any events. self.report.report_failure( - "
", + "No Metadata Produced", "No metadata was produced. Check the logs for more details.", ) @@ -2024,6 +670,7 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 self.source_config.project_name, self.base_projects_folder, self.reporter, + self.source_config.liquid_variable, ) # Some views can be mentioned by multiple 'include' statements and can be included via different connections. @@ -2057,19 +704,26 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 model = self._load_model(str(file_path)) except Exception as e: self.reporter.report_warning( - model_name, f"unable to load Looker model at {file_path}: {repr(e)}" + title="Error Loading Model File", + message="Unable to load Looker model from file.", + context=f"Model Name: {model_name}, File Path: {file_path}", + exc=e, ) continue assert model.connection is not None - connectionDefinition = self._get_connection_def_based_on_connection_string( - model.connection + connection_definition = get_connection_def_based_on_connection_string( + connection=model.connection, + looker_client=self.looker_client, + source_config=self.source_config, + reporter=self.reporter, ) - if connectionDefinition is None: + if connection_definition is None: self.reporter.report_warning( - f"model-{model_name}", - f"Failed to load connection {model.connection}. Check your API key permissions and/or connection_to_platform_map configuration.", + title="Failed to Load Connection", + message="Failed to load connection. Check your API key permissions and/or connection_to_platform_map configuration.", + context=f"Connection: {model.connection}", ) self.reporter.report_models_dropped(model_name) continue @@ -2078,12 +732,13 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 looker_refinement_resolver: LookerRefinementResolver = ( LookerRefinementResolver( looker_model=model, - connection_definition=connectionDefinition, + connection_definition=connection_definition, looker_viewfile_loader=viewfile_loader, source_config=self.source_config, reporter=self.reporter, ) ) + if self.source_config.emit_reachable_views_only: model_explores_map = {d["name"]: d for d in model.explores} for explore_dict in model.explores: @@ -2109,8 +764,10 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 explore_reachable_views.add(view_name.include) except Exception as e: self.reporter.report_warning( - f"{model}.explores", - f"failed to process {explore_dict} due to {e}. Run with --debug for full stacktrace", + title="Failed to process explores", + message="Failed to process explore dictionary.", + context=f"Explore Details: {explore_dict}", + exc=e, ) logger.debug("Failed to process explore", exc_info=e) @@ -2120,6 +777,14 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 project_name = self.get_project_name(model_name) + looker_view_id_cache: LookerViewIdCache = LookerViewIdCache( + project_name=project_name, + model_name=model_name, + looker_model=model, + looker_viewfile_loader=viewfile_loader, + reporter=self.reporter, + ) + logger.debug(f"Model: {model_name}; Includes: {model.resolved_includes}") for include in model.resolved_includes: @@ -2131,7 +796,7 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 looker_viewfile = viewfile_loader.load_viewfile( path=include.include, project_name=include.project, - connection=connectionDefinition, + connection=connection_definition, reporter=self.reporter, ) @@ -2163,7 +828,8 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 else project_name ) - # if project is base project then it is available as self.base_projects_folder[_BASE_PROJECT_NAME] + # if project is base project then it is available as self.base_projects_folder[ + # _BASE_PROJECT_NAME] base_folder_path: str = str( self.base_projects_folder.get( current_project_name, @@ -2171,28 +837,38 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 ) ) + view_context: LookerViewContext = LookerViewContext( + raw_view=raw_view, + view_file=looker_viewfile, + view_connection=connection_definition, + view_file_loader=viewfile_loader, + looker_refinement_resolver=looker_refinement_resolver, + base_folder_path=base_folder_path, + reporter=self.reporter, + ) + maybe_looker_view = LookerView.from_looker_dict( project_name=current_project_name, - base_folder_path=base_folder_path, model_name=model_name, - looker_view=raw_view, - connection=connectionDefinition, - looker_viewfile=looker_viewfile, - looker_viewfile_loader=viewfile_loader, - looker_refinement_resolver=looker_refinement_resolver, + view_context=view_context, + looker_view_id_cache=looker_view_id_cache, reporter=self.reporter, max_file_snippet_length=self.source_config.max_file_snippet_length, - parse_table_names_from_sql=self.source_config.parse_table_names_from_sql, - sql_parser_path=self.source_config.sql_parser, extract_col_level_lineage=self.source_config.extract_column_level_lineage, populate_sql_logic_in_descriptions=self.source_config.populate_sql_logic_for_missing_descriptions, - process_isolation_for_sql_parsing=self.source_config.process_isolation_for_sql_parsing, + config=self.source_config, + ctx=self.ctx, ) except Exception as e: self.reporter.report_warning( - include.include, - f"unable to load Looker view {raw_view}: {repr(e)}", + title="Error Loading View", + message="Unable to load Looker View.", + context=f"View Details: {raw_view}", + exc=e, ) + + logger.debug(e, exc_info=e) + continue if maybe_looker_view: @@ -2243,7 +919,8 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 prev_model_connection, ) = view_connection_mapping if prev_model_connection != model.connection: - # this view has previously been discovered and emitted using a different connection + # this view has previously been discovered and emitted using a different + # connection logger.warning( f"view {maybe_looker_view.id.view_name} from model {model_name}, connection {model.connection} was previously processed via model {prev_model_name}, connection {prev_model_connection} and will likely lead to incorrect lineage to the underlying tables" ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/str_functions.py b/metadata-ingestion/src/datahub/ingestion/source/looker/str_functions.py new file mode 100644 index 0000000000000..5426d2b8ab952 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/str_functions.py @@ -0,0 +1,23 @@ +""" +Here write down functions which are operating on string. Like replacing some character and so on +""" +import re + + +def remove_suffix(original: str, suffix: str) -> str: + # This can be removed in favour of original.removesuffix for python>3.8 + if original.endswith(suffix): + return original[: -len(suffix)] + return original + + +def remove_extra_spaces_and_newlines(original: str) -> str: + """ + python-liquid library is not removing extra spaces and new lines from template and hence spaces and newlines + are appearing in urn. This function can be used to remove such characters from urn or text. + """ + return re.sub(r"\s*\n\s*", "", original) + + +def replace_quotes(value: str) -> str: + return value.replace('"', "").replace("`", "") diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/urn_functions.py b/metadata-ingestion/src/datahub/ingestion/source/looker/urn_functions.py new file mode 100644 index 0000000000000..7286beb1f977a --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/urn_functions.py @@ -0,0 +1,18 @@ +def get_qualified_table_name(urn: str) -> str: + part: str = urn.split(",")[-2] + + if len(part.split(".")) >= 4: + return ".".join( + part.split(".")[-3:] + ) # return only db.schema.table skip platform instance as higher code is + # failing if encounter platform-instance in qualified table name + else: + return part + + +def get_table_name(urn: str) -> str: + qualified_table_name: str = get_qualified_table_name( + urn=urn, + ) + + return qualified_table_name.split(".")[-1] diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py b/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py new file mode 100644 index 0000000000000..390e71ef9d4bd --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py @@ -0,0 +1,636 @@ +import logging +import re +from abc import ABC, abstractmethod +from functools import lru_cache +from typing import Dict, List, Optional + +from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.source.looker.looker_common import ( + LookerExplore, + LookerViewId, + ViewField, + ViewFieldType, +) +from datahub.ingestion.source.looker.looker_view_id_cache import LookerViewIdCache +from datahub.ingestion.source.looker.lookml_concept_context import ( + LookerFieldContext, + LookerViewContext, +) +from datahub.ingestion.source.looker.lookml_config import ( + DERIVED_VIEW_SUFFIX, + NAME, + LookerConnectionDefinition, + LookMLSourceConfig, + LookMLSourceReport, +) +from datahub.ingestion.source.looker.urn_functions import get_qualified_table_name +from datahub.sql_parsing.sqlglot_lineage import ( + ColumnLineageInfo, + ColumnRef, + SqlParsingResult, + Urn, + create_lineage_sql_parsed_result, +) + +logger = logging.getLogger(__name__) + + +def is_derived_view(view_name: str) -> bool: + if DERIVED_VIEW_SUFFIX in view_name.lower(): + return True + + return False + + +def get_derived_looker_view_id( + qualified_table_name: str, + looker_view_id_cache: "LookerViewIdCache", + base_folder_path: str, +) -> Optional[LookerViewId]: + # qualified_table_name can be in either of below format + # 1) db.schema.employee_income_source.sql_table_name + # 2) db.employee_income_source.sql_table_name + # 3) employee_income_source.sql_table_name + # In any of the form we need the text coming before ".sql_table_name" and after last "." + parts: List[str] = re.split( + DERIVED_VIEW_SUFFIX, qualified_table_name, flags=re.IGNORECASE + ) + view_name: str = parts[0].split(".")[-1] + + looker_view_id: Optional[LookerViewId] = looker_view_id_cache.get_looker_view_id( + view_name=view_name, + base_folder_path=base_folder_path, + ) + + return looker_view_id + + +def resolve_derived_view_urn_of_col_ref( + column_refs: List[ColumnRef], + looker_view_id_cache: "LookerViewIdCache", + base_folder_path: str, + config: LookMLSourceConfig, +) -> List[ColumnRef]: + + new_column_refs: List[ColumnRef] = [] + for col_ref in column_refs: + if is_derived_view(col_ref.table.lower()): + new_urns: List[str] = fix_derived_view_urn( + urns=[col_ref.table], + looker_view_id_cache=looker_view_id_cache, + base_folder_path=base_folder_path, + config=config, + ) + if not new_urns: + logger.warning( + f"Not able to resolve to derived view looker id for {col_ref.table}" + ) + continue + + new_column_refs.append(ColumnRef(table=new_urns[0], column=col_ref.column)) + else: + new_column_refs.append(col_ref) + + return new_column_refs + + +def fix_derived_view_urn( + urns: List[str], + looker_view_id_cache: "LookerViewIdCache", + base_folder_path: str, + config: LookMLSourceConfig, +) -> List[str]: + # Regenerate view urn if .sql_table_name is present in urn + new_urns: List[str] = [] + for urn in urns: + if is_derived_view(urn): + looker_view_id = get_derived_looker_view_id( + qualified_table_name=get_qualified_table_name(urn), + looker_view_id_cache=looker_view_id_cache, + base_folder_path=base_folder_path, + ) + + if looker_view_id is None: + logger.warning( + f"Not able to resolve to derived view looker id for {urn}" + ) + continue + + new_urns.append(looker_view_id.get_urn(config=config)) + else: + new_urns.append(urn) + + return new_urns + + +def _platform_names_have_2_parts(platform: str) -> bool: + return platform in {"hive", "mysql", "athena"} + + +def _drop_hive_dot(urn: str) -> str: + """ + This is special handling for hive platform where "hive." is coming in urn's id because of the way SQL + is written in lookml. + + Example: urn:li:dataset:(urn:li:dataPlatform:hive,hive.my_database.my_table,PROD) + + Here we need to transform hive.my_database.my_table to my_database.my_table + """ + if urn.startswith("urn:li:dataset:(urn:li:dataPlatform:hive"): + return re.sub(r"hive\.", "", urn) + + return urn + + +def _drop_hive_dot_from_upstream(upstreams: List[ColumnRef]) -> List[ColumnRef]: + return [ + ColumnRef(table=_drop_hive_dot(column_ref.table), column=column_ref.column) + for column_ref in upstreams + ] + + +def _generate_fully_qualified_name( + sql_table_name: str, + connection_def: LookerConnectionDefinition, + reporter: LookMLSourceReport, +) -> str: + """Returns a fully qualified dataset name, resolved through a connection definition. + Input sql_table_name can be in three forms: table, db.table, db.schema.table""" + # TODO: This function should be extracted out into a Platform specific naming class since name translations + # are required across all connectors + + # Bigquery has "project.db.table" which can be mapped to db.schema.table form + # All other relational db's follow "db.schema.table" + # With the exception of mysql, hive, athena which are "db.table" + + # first detect which one we have + parts = len(sql_table_name.split(".")) + + if parts == 3: + # fully qualified, but if platform is of 2-part, we drop the first level + if _platform_names_have_2_parts(connection_def.platform): + sql_table_name = ".".join(sql_table_name.split(".")[1:]) + return sql_table_name.lower() + + if parts == 1: + # Bare table form + if _platform_names_have_2_parts(connection_def.platform): + dataset_name = f"{connection_def.default_db}.{sql_table_name}" + else: + dataset_name = f"{connection_def.default_db}.{connection_def.default_schema}.{sql_table_name}" + return dataset_name.lower() + + if parts == 2: + # if this is a 2 part platform, we are fine + if _platform_names_have_2_parts(connection_def.platform): + return sql_table_name.lower() + # otherwise we attach the default top-level container + dataset_name = f"{connection_def.default_db}.{sql_table_name}" + return dataset_name.lower() + + reporter.report_warning( + title="Malformed Table Name", + message="Table name has more than 3 parts.", + context=f"Table Name: {sql_table_name}", + ) + return sql_table_name.lower() + + +class AbstractViewUpstream(ABC): + """ + Implementation of this interface extracts the view upstream as per the way the view is bound to datasets. + For detail explanation please refer lookml_concept_context.LookerViewContext documentation. + """ + + view_context: LookerViewContext + looker_view_id_cache: LookerViewIdCache + config: LookMLSourceConfig + ctx: PipelineContext + + def __init__( + self, + view_context: LookerViewContext, + looker_view_id_cache: LookerViewIdCache, + config: LookMLSourceConfig, + ctx: PipelineContext, + ): + self.view_context = view_context + self.looker_view_id_cache = looker_view_id_cache + self.config = config + self.ctx = ctx + + @abstractmethod + def get_upstream_column_ref( + self, field_context: LookerFieldContext + ) -> List[ColumnRef]: + pass + + @abstractmethod + def get_upstream_dataset_urn(self) -> List[Urn]: + pass + + def create_fields(self) -> List[ViewField]: + return [] # it is for the special case + + +class SqlBasedDerivedViewUpstream(AbstractViewUpstream): + """ + Handle the case where upstream dataset is defined in derived_table.sql + """ + + def __init__( + self, + view_context: LookerViewContext, + looker_view_id_cache: LookerViewIdCache, + config: LookMLSourceConfig, + ctx: PipelineContext, + ): + super().__init__(view_context, looker_view_id_cache, config, ctx) + # These are the function where we need to catch the response once calculated + self._get_spr = lru_cache(maxsize=1)(self.__get_spr) + self._get_upstream_dataset_urn = lru_cache(maxsize=1)( + self.__get_upstream_dataset_urn + ) + + def __get_spr(self) -> Optional[SqlParsingResult]: + # for backward compatibility + if not self.config.parse_table_names_from_sql: + return None + + spr = create_lineage_sql_parsed_result( + query=self.view_context.sql(), + default_schema=self.view_context.view_connection.default_schema, + default_db=self.view_context.view_connection.default_db, + platform=self.view_context.view_connection.platform, + platform_instance=self.view_context.view_connection.platform_instance, + env=self.view_context.view_connection.platform_env or self.config.env, + graph=self.ctx.graph, + ) + + if ( + spr.debug_info.table_error is not None + or spr.debug_info.column_error is not None + ): + logging.debug( + f"Failed to parsed the sql query. table_error={spr.debug_info.table_error} and " + f"column_error={spr.debug_info.column_error}" + ) + return None + + return spr + + def __get_upstream_dataset_urn(self) -> List[Urn]: + sql_parsing_result: Optional[SqlParsingResult] = self._get_spr() + + if sql_parsing_result is None: + return [] + + upstream_dataset_urns: List[str] = [ + _drop_hive_dot(urn) for urn in sql_parsing_result.in_tables + ] + + # fix any derived view reference present in urn + upstream_dataset_urns = fix_derived_view_urn( + urns=upstream_dataset_urns, + looker_view_id_cache=self.looker_view_id_cache, + base_folder_path=self.view_context.base_folder_path, + config=self.config, + ) + + return upstream_dataset_urns + + def create_fields(self) -> List[ViewField]: + spr: Optional[SqlParsingResult] = self._get_spr() + + if spr is None: + return [] + + fields: List[ViewField] = [] + + column_lineages: List[ColumnLineageInfo] = ( + spr.column_lineage if spr.column_lineage is not None else [] + ) + + for cll in column_lineages: + fields.append( + ViewField( + name=cll.downstream.column, + label="", + type=cll.downstream.native_column_type + if cll.downstream.native_column_type is not None + else "unknown", + description="", + field_type=ViewFieldType.UNKNOWN, + upstream_fields=_drop_hive_dot_from_upstream(cll.upstreams), + ) + ) + + return fields + + def get_upstream_column_ref( + self, field_context: LookerFieldContext + ) -> List[ColumnRef]: + sql_parsing_result: Optional[SqlParsingResult] = self._get_spr() + + if sql_parsing_result is None: + return [] + + upstreams_column_refs: List[ColumnRef] = [] + if sql_parsing_result.column_lineage: + for cll in sql_parsing_result.column_lineage: + if cll.downstream.column == field_context.name(): + upstreams_column_refs = cll.upstreams + break + + # field might get skip either because of Parser not able to identify the column from GMS + # in-case of "select * from look_ml_view.SQL_TABLE_NAME" or extra field are defined in the looker view which is + # referring to upstream table + if self._get_upstream_dataset_urn() and not upstreams_column_refs: + upstreams_column_refs = [ + ColumnRef( + table=self._get_upstream_dataset_urn()[ + 0 + ], # 0th index has table of from clause + column=column, + ) + for column in field_context.column_name_in_sql_attribute() + ] + + # fix any derived view reference present in urn + upstreams_column_refs = resolve_derived_view_urn_of_col_ref( + column_refs=upstreams_column_refs, + looker_view_id_cache=self.looker_view_id_cache, + base_folder_path=self.view_context.base_folder_path, + config=self.config, + ) + + return upstreams_column_refs + + def get_upstream_dataset_urn(self) -> List[Urn]: + return self._get_upstream_dataset_urn() + + +class NativeDerivedViewUpstream(AbstractViewUpstream): + """ + Handle the case where upstream dataset is defined as derived_table.explore_source + """ + + upstream_dataset_urns: List[str] + explore_column_mapping: Dict + + def __init__( + self, + view_context: LookerViewContext, + looker_view_id_cache: LookerViewIdCache, + config: LookMLSourceConfig, + ctx: PipelineContext, + ): + super().__init__(view_context, looker_view_id_cache, config, ctx) + self._get_upstream_dataset_urn = lru_cache(maxsize=1)( + self.__get_upstream_dataset_urn + ) + self._get_explore_column_mapping = lru_cache(maxsize=1)( + self.__get_explore_column_mapping + ) + + def __get_upstream_dataset_urn(self) -> List[str]: + current_view_id: Optional[ + LookerViewId + ] = self.looker_view_id_cache.get_looker_view_id( + view_name=self.view_context.name(), + base_folder_path=self.view_context.base_folder_path, + ) + + # Current view will always be present in cache. The assert will silence the lint + assert current_view_id + + # We're creating a "LookerExplore" just to use the urn generator. + upstream_dataset_urns: List[str] = [ + LookerExplore( + name=self.view_context.explore_source()[NAME], + model_name=current_view_id.model_name, + ).get_explore_urn(self.config) + ] + + return upstream_dataset_urns + + def __get_explore_column_mapping(self) -> Dict: + explore_columns: Dict = self.view_context.explore_source().get("columns", {}) + + explore_column_mapping = {} + + for column in explore_columns: + explore_column_mapping[column[NAME]] = column + + return explore_column_mapping + + def get_upstream_column_ref( + self, field_context: LookerFieldContext + ) -> List[ColumnRef]: + upstream_column_refs: List[ColumnRef] = [] + + if not self._get_upstream_dataset_urn(): + # No upstream explore dataset found + logging.debug( + f"upstream explore not found for field {field_context.name()} of view {self.view_context.name()}" + ) + return upstream_column_refs + + explore_urn: str = self._get_upstream_dataset_urn()[0] + + for column in field_context.column_name_in_sql_attribute(): + if column in self._get_explore_column_mapping(): + explore_column: Dict = self._get_explore_column_mapping()[column] + upstream_column_refs.append( + ColumnRef( + column=explore_column.get("field", explore_column[NAME]), + table=explore_urn, + ) + ) + + return upstream_column_refs + + def get_upstream_dataset_urn(self) -> List[Urn]: + return self._get_upstream_dataset_urn() + + +class RegularViewUpstream(AbstractViewUpstream): + """ + Handle the case where upstream dataset name is equal to view-name + """ + + upstream_dataset_urn: Optional[str] + + def __init__( + self, + view_context: LookerViewContext, + looker_view_id_cache: LookerViewIdCache, + config: LookMLSourceConfig, + ctx: PipelineContext, + ): + super().__init__(view_context, looker_view_id_cache, config, ctx) + self.upstream_dataset_urn = None + + self._get_upstream_dataset_urn = lru_cache(maxsize=1)( + self.__get_upstream_dataset_urn + ) + + def __get_upstream_dataset_urn(self) -> Urn: + # In regular case view's upstream dataset is either same as view-name or mentioned in "sql_table_name" field + # view_context.sql_table_name() handle this condition to return dataset name + qualified_table_name: str = _generate_fully_qualified_name( + sql_table_name=self.view_context.sql_table_name(), + connection_def=self.view_context.view_connection, + reporter=self.view_context.reporter, + ) + + self.upstream_dataset_urn = make_dataset_urn_with_platform_instance( + platform=self.view_context.view_connection.platform, + name=qualified_table_name.lower(), + platform_instance=self.view_context.view_connection.platform_instance, + env=self.view_context.view_connection.platform_env or self.config.env, + ) + + return self.upstream_dataset_urn + + def get_upstream_column_ref( + self, field_context: LookerFieldContext + ) -> List[ColumnRef]: + upstream_column_ref: List[ColumnRef] = [] + + for column_name in field_context.column_name_in_sql_attribute(): + upstream_column_ref.append( + ColumnRef(table=self._get_upstream_dataset_urn(), column=column_name) + ) + + return upstream_column_ref + + def get_upstream_dataset_urn(self) -> List[Urn]: + return [self._get_upstream_dataset_urn()] + + +class DotSqlTableNameViewUpstream(AbstractViewUpstream): + """ + Handle the case where upstream dataset name is mentioned as sql_table_name: ${view-name.SQL_TABLE_NAME} + """ + + upstream_dataset_urn: List[Urn] + + def __init__( + self, + view_context: LookerViewContext, + looker_view_id_cache: LookerViewIdCache, + config: LookMLSourceConfig, + ctx: PipelineContext, + ): + super().__init__(view_context, looker_view_id_cache, config, ctx) + self.upstream_dataset_urn = [] + + self._get_upstream_dataset_urn = lru_cache(maxsize=1)( + self.__get_upstream_dataset_urn + ) + + def __get_upstream_dataset_urn(self) -> List[Urn]: + # In this case view_context.sql_table_name() refers to derived view name + looker_view_id = get_derived_looker_view_id( + qualified_table_name=_generate_fully_qualified_name( + self.view_context.sql_table_name(), + self.view_context.view_connection, + self.view_context.reporter, + ), + base_folder_path=self.view_context.base_folder_path, + looker_view_id_cache=self.looker_view_id_cache, + ) + + if looker_view_id is not None: + self.upstream_dataset_urn = [ + looker_view_id.get_urn( + config=self.config, + ) + ] + + return self.upstream_dataset_urn + + def get_upstream_column_ref( + self, field_context: LookerFieldContext + ) -> List[ColumnRef]: + upstream_column_ref: List[ColumnRef] = [] + if not self._get_upstream_dataset_urn(): + return upstream_column_ref + + for column_name in field_context.column_name_in_sql_attribute(): + upstream_column_ref.append( + ColumnRef(table=self._get_upstream_dataset_urn()[0], column=column_name) + ) + + return upstream_column_ref + + def get_upstream_dataset_urn(self) -> List[Urn]: + return self._get_upstream_dataset_urn() + + +class EmptyImplementation(AbstractViewUpstream): + def get_upstream_column_ref( + self, field_context: LookerFieldContext + ) -> List[ColumnRef]: + return [] + + def get_upstream_dataset_urn(self) -> List[Urn]: + return [] + + +def create_view_upstream( + view_context: LookerViewContext, + looker_view_id_cache: LookerViewIdCache, + config: LookMLSourceConfig, + ctx: PipelineContext, + reporter: LookMLSourceReport, +) -> AbstractViewUpstream: + if view_context.is_regular_case(): + return RegularViewUpstream( + view_context=view_context, + config=config, + ctx=ctx, + looker_view_id_cache=looker_view_id_cache, + ) + + if view_context.is_sql_table_name_referring_to_view(): + return DotSqlTableNameViewUpstream( + view_context=view_context, + config=config, + ctx=ctx, + looker_view_id_cache=looker_view_id_cache, + ) + + if ( + view_context.is_sql_based_derived_case() + or view_context.is_sql_based_derived_view_without_fields_case() + ): + return SqlBasedDerivedViewUpstream( + view_context=view_context, + config=config, + ctx=ctx, + looker_view_id_cache=looker_view_id_cache, + ) + + if view_context.is_native_derived_case(): + return NativeDerivedViewUpstream( + view_context=view_context, + config=config, + ctx=ctx, + looker_view_id_cache=looker_view_id_cache, + ) + + reporter.report_warning( + title="Implementation Not Found", + message="No implementation found to resolve upstream of the view", + context=view_context.view_file_name(), + ) + + return EmptyImplementation( + view_context=view_context, + config=config, + ctx=ctx, + looker_view_id_cache=looker_view_id_cache, + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/metabase.py b/metadata-ingestion/src/datahub/ingestion/source/metabase.py index 12a76ff7b33ff..49fa9dab5f1d8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/metabase.py +++ b/metadata-ingestion/src/datahub/ingestion/source/metabase.py @@ -211,8 +211,8 @@ def setup_session(self) -> None: test_response.raise_for_status() except HTTPError as e: self.report.report_failure( - key="metabase-session", - reason=f"Unable to retrieve user {self.config.username} information. %s" + title="Unable to Retrieve Current User", + message=f"Unable to retrieve user {self.config.username} information. %s" % str(e), ) @@ -223,8 +223,8 @@ def close(self) -> None: ) if response.status_code not in (200, 204): self.report.report_failure( - key="metabase-session", - reason=f"Unable to logout for user {self.config.username}", + title="Unable to Log User Out", + message=f"Unable to logout for user {self.config.username}", ) super().close() @@ -257,8 +257,9 @@ def emit_dashboard_mces(self) -> Iterable[MetadataWorkUnit]: except HTTPError as http_error: self.report.report_failure( - key="metabase-dashboard", - reason=f"Unable to retrieve dashboards. " f"Reason: {str(http_error)}", + title="Unable to Retrieve Dashboards", + message="Request to retrieve dashboards from Metabase failed.", + context=f"Error: {str(http_error)}", ) @staticmethod @@ -283,8 +284,9 @@ def construct_dashboard_from_api_data( dashboard_details = dashboard_response.json() except HTTPError as http_error: self.report.report_warning( - key=f"metabase-dashboard-{dashboard_id}", - reason=f"Unable to retrieve dashboard. " f"Reason: {str(http_error)}", + title="Unable to Retrieve Dashboard", + message="Request to retrieve dashboards from Metabase failed.", + context=f"Dashboard ID: {dashboard_id}, Error: {str(http_error)}", ) return None @@ -346,14 +348,16 @@ def _get_ownership(self, creator_id: int) -> Optional[OwnershipClass]: and http_error.response.status_code == 404 ): self.report.report_warning( - key=f"metabase-user-{creator_id}", - reason=f"User {creator_id} is blocked in Metabase or missing", + title="Cannot find user", + message="User is blocked in Metabase or missing", + context=f"Creator ID: {creator_id}", ) return None # For cases when the error is not 404 but something else self.report.report_warning( - key=f"metabase-user-{creator_id}", - reason=f"Unable to retrieve User info. " f"Reason: {str(http_error)}", + title="Failed to retrieve user", + message="Request to Metabase Failed", + context=f"Creator ID: {creator_id}, Error: {str(http_error)}", ) return None @@ -385,8 +389,9 @@ def emit_card_mces(self) -> Iterable[MetadataWorkUnit]: except HTTPError as http_error: self.report.report_failure( - key="metabase-cards", - reason=f"Unable to retrieve cards. " f"Reason: {str(http_error)}", + title="Unable to Retrieve Cards", + message="Request to retrieve cards from Metabase failed.", + context=f"Error: {str(http_error)}", ) return None @@ -407,8 +412,9 @@ def get_card_details_by_id(self, card_id: Union[int, str]) -> dict: return card_response.json() except HTTPError as http_error: self.report.report_warning( - key=f"metabase-card-{card_id}", - reason=f"Unable to retrieve Card info. " f"Reason: {str(http_error)}", + title="Unable to Retrieve Card", + message="Request to retrieve Card from Metabase failed.", + context=f"Card ID: {card_id}, Error: {str(http_error)}", ) return {} @@ -416,16 +422,18 @@ def construct_card_from_api_data(self, card_data: dict) -> Optional[ChartSnapsho card_id = card_data.get("id") if card_id is None: self.report.report_warning( - key="metabase-card", - reason=f"Unable to get Card id from card data {str(card_data)}", + title="Card is missing 'id'", + message="Unable to get field id from card data.", + context=f"Card Details: {str(card_data)}", ) return None card_details = self.get_card_details_by_id(card_id) if not card_details: self.report.report_warning( - key=f"metabase-card-{card_id}", - reason="Unable to construct Card due to empty card details", + title="Missing Card Details", + message="Unable to construct Card due to empty card details", + context=f"Card ID: {card_id}", ) return None @@ -441,7 +449,7 @@ def construct_card_from_api_data(self, card_data: dict) -> Optional[ChartSnapsho f"{last_edit_by.get('timestamp')}" ) last_modified = ChangeAuditStamps( - created=AuditStamp(time=modified_ts, actor=modified_actor), + created=None, lastModified=AuditStamp(time=modified_ts, actor=modified_actor), ) @@ -500,16 +508,18 @@ def _get_chart_type(self, card_id: int, display_type: str) -> Optional[str]: } if not display_type: self.report.report_warning( - key=f"metabase-card-{card_id}", - reason=f"Card type {display_type} is missing. Setting to None", + title="Unrecognized Card Type", + message=f"Unrecognized card type {display_type} found. Setting to None", + context=f"Card ID: {card_id}", ) return None try: chart_type = type_mapping[display_type] except KeyError: self.report.report_warning( - key=f"metabase-card-{card_id}", - reason=f"Chart type {display_type} not supported. Setting to None", + title="Unrecognized Chart Type", + message=f"Unrecognized chart type {display_type} found. Setting to None", + context=f"Card ID: {card_id}", ) chart_type = None @@ -543,8 +553,9 @@ def get_datasource_urn( ) -> Optional[List]: if recursion_depth > DATASOURCE_URN_RECURSION_LIMIT: self.report.report_warning( - key=f"metabase-card-{card_details.get('id')}", - reason="Unable to retrieve Card info. Reason: source table recursion depth exceeded", + title="Unable to Retrieve Card Info", + message="Unable to retrieve Card info. Source table recursion depth exceeded.", + context=f"Card Details: {card_details}", ) return None @@ -557,8 +568,9 @@ def get_datasource_urn( ) = self.get_datasource_from_id(datasource_id) if not platform: self.report.report_warning( - key=f"metabase-datasource-{datasource_id}", - reason=f"Unable to detect platform for database id {datasource_id}", + title="Unable to find Data Platform", + message="Unable to detect Data Platform for database id", + context=f"Data Source ID: {datasource_id}", ) return None @@ -613,8 +625,9 @@ def get_datasource_urn( f"{result.debug_info.table_error}" ) self.report.report_warning( - key="metabase-query", - reason=f"Unable to retrieve lineage from query: {raw_query_stripped}", + title="Failed to Extract Lineage", + message="Unable to retrieve lineage from query", + context=f"Query: {raw_query_stripped}", ) return result.in_tables @@ -654,8 +667,9 @@ def get_source_table_from_id( except HTTPError as http_error: self.report.report_warning( - key=f"metabase-table-{table_id}", - reason=f"Unable to retrieve source table. Reason: {str(http_error)}", + title="Failed to Retrieve Source Table", + message="Request to retrieve source table from Metadabase failed", + context=f"Table ID: {table_id}, Error: {str(http_error)}", ) return None, None @@ -702,8 +716,9 @@ def get_datasource_from_id( dataset_json = dataset_response.json() except HTTPError as http_error: self.report.report_warning( - key=f"metabase-datasource-{datasource_id}", - reason=f"Unable to retrieve Datasource. " f"Reason: {str(http_error)}", + title="Unable to Retrieve Data Source", + message="Request to retrieve data source from Metabase failed.", + context=f"Data Source ID: {datasource_id}, Error: {str(http_error)}", ) # returning empty string as `platform` because # `make_dataset_urn_with_platform_instance()` only accepts `str` @@ -730,8 +745,9 @@ def get_datasource_from_id( platform = engine self.report.report_warning( - key=f"metabase-platform-{datasource_id}", - reason=f"Platform was not found in DataHub. Using {platform} name as is", + title="Unrecognized Data Platform found", + message="Data Platform was not found. Using platform name as is", + context=f"Platform: {platform}", ) platform_instance = self.get_platform_instance( @@ -766,8 +782,9 @@ def get_datasource_from_id( dbname = self.config.database_alias_map[platform] else: self.report.report_warning( - key=f"metabase-dbname-{datasource_id}", - reason=f"Cannot determine database name for platform: {platform}", + title="Cannot resolve Database Name", + message="Cannot determine database name for platform", + context=f"Platform: {platform}", ) return platform, dbname, schema, platform_instance diff --git a/metadata-ingestion/src/datahub/ingestion/source/mode.py b/metadata-ingestion/src/datahub/ingestion/source/mode.py index 4aa6248d09ab3..4b4822bcb98ca 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/mode.py +++ b/metadata-ingestion/src/datahub/ingestion/source/mode.py @@ -311,8 +311,9 @@ def __init__(self, ctx: PipelineContext, config: ModeConfig): self._get_request_json(f"{self.config.connect_uri}/api/verify") except HTTPError as http_error: self.report.report_failure( - key="mode-session", - reason=f"Unable to verify connection. Error was: {str(http_error)}", + title="Failed to Connect", + message="Unable to verify connection to mode.", + context=f"Error: {str(http_error)}", ) self.workspace_uri = f"{self.config.connect_uri}/api/{self.config.workspace}" @@ -371,15 +372,15 @@ def construct_dashboard( if not report_token: self.report.report_warning( - key="mode-report", - reason=f"Report token is missing for {report_info.get('id', '')}", + title="Missing Report Token", + message=f"Report token is missing for {report_info.get('id', '')}", ) return None if not report_info.get("id"): self.report.report_warning( - key="mode-report", - reason=f"Report id is missing for {report_info.get('token', '')}", + title="Missing Report ID", + message=f"Report id is missing for {report_info.get('token', '')}", ) return None @@ -488,9 +489,9 @@ def _get_creator(self, href: str) -> Optional[str]: ) except HTTPError as http_error: self.report.report_warning( - key="mode-user", - reason=f"Unable to retrieve user for {href}, " - f"Reason: {str(http_error)}", + title="Failed to retrieve Mode creator", + message=f"Unable to retrieve user for {href}", + context=f"Reason: {str(http_error)}", ) return user @@ -528,9 +529,9 @@ def _get_space_name_and_tokens(self) -> dict: space_info[s.get("token", "")] = s.get("name", "") except HTTPError as http_error: self.report.report_failure( - key="mode-spaces", - reason=f"Unable to retrieve spaces/collections for {self.workspace_uri}, " - f"Reason: {str(http_error)}", + title="Failed to Retrieve Spaces", + message="Unable to retrieve spaces / collections for workspace.", + context=f"Workspace: {self.workspace_uri}, Error: {str(http_error)}", ) return space_info @@ -559,17 +560,19 @@ def _get_chart_type(self, token: str, display_type: str) -> Optional[str]: "histogram": ChartTypeClass.HISTOGRAM, } if not display_type: - self.report.report_warning( - key="mode-chart-type-mapper", - reason=f"{token}: Chart type is missing. Setting to None", + self.report.info( + title="Missing chart type found", + message="Chart type is missing. Setting to None", + context=f"Token: {token}", ) return None try: chart_type = type_mapping[display_type] except KeyError: - self.report.report_warning( - key="mode-chart-type-mapper", - reason=f"{token}: Chart type {display_type} not supported. Setting to None", + self.report.info( + title="Unrecognized chart type found", + message=f"Chart type {display_type} not supported. Setting to None", + context=f"Token: {token}", ) chart_type = None @@ -661,8 +664,8 @@ def _get_datahub_friendly_platform(self, adapter, platform): return platform_mapping[adapter] else: self.report.report_warning( - key=f"mode-platform-{adapter}", - reason=f"Platform was not found in DataHub. " + title="Unrecognized Platform Found", + message=f"Platform was not found in DataHub. " f"Using {platform} name as is", ) @@ -676,8 +679,9 @@ def _get_data_sources(self) -> List[dict]: data_sources = ds_json.get("_embedded", {}).get("data_sources", []) except HTTPError as http_error: self.report.report_failure( - key="mode-data-sources", - reason=f"Unable to retrieve data sources. Reason: {str(http_error)}", + title="Failed to retrieve Data Sources", + message="Unable to retrieve data sources from Mode.", + context=f"Error: {str(http_error)}", ) return data_sources @@ -690,8 +694,9 @@ def _get_platform_and_dbname( if not data_sources: self.report.report_failure( - key=f"mode-datasource-{data_source_id}", - reason=f"No data sources found for datasource id: " f"{data_source_id}", + title="No Data Sources Found", + message="Could not find data sources matching some ids", + context=f"Data Soutce ID: {data_source_id}", ) return None, None @@ -704,8 +709,8 @@ def _get_platform_and_dbname( return platform, database else: self.report.report_warning( - key=f"mode-datasource-{data_source_id}", - reason=f"Cannot create datasource urn for datasource id: " + title="Failed to create Data Platform Urn", + message=f"Cannot create datasource urn for datasource id: " f"{data_source_id}", ) return None, None @@ -761,9 +766,9 @@ def _get_definition(self, definition_name): except HTTPError as http_error: self.report.report_failure( - key=f"mode-definition-{definition_name}", - reason=f"Unable to retrieve definition for {definition_name}, " - f"Reason: {str(http_error)}", + title="Failed to Retrieve Definition", + message="Unable to retrieve definition from Mode.", + context=f"Definition Name: {definition_name}, Error: {str(http_error)}", ) return None @@ -782,10 +787,9 @@ def _get_source_from_query(self, raw_query: str) -> set: source_paths.add(f"{source_schema}.{source_table}") except Exception as e: self.report.report_failure( - key="mode-query", - reason=f"Unable to retrieve lineage from query. " - f"Query: {raw_query} " - f"Reason: {str(e)} ", + title="Failed to Extract Lineage From Query", + message="Unable to retrieve lineage from Mode query.", + context=f"Query: {raw_query}, Error: {str(e)}", ) return source_paths @@ -1321,9 +1325,9 @@ def _get_reports(self, space_token: str) -> List[dict]: reports = reports_json.get("_embedded", {}).get("reports", {}) except HTTPError as http_error: self.report.report_failure( - key=f"mode-report-{space_token}", - reason=f"Unable to retrieve reports for space token: {space_token}, " - f"Reason: {str(http_error)}", + title="Failed to Retrieve Reports for Space", + message="Unable to retrieve reports for space token.", + context=f"Space Token: {space_token}, Error: {str(http_error)}", ) return reports @@ -1337,9 +1341,9 @@ def _get_queries(self, report_token: str) -> list: queries = queries_json.get("_embedded", {}).get("queries", {}) except HTTPError as http_error: self.report.report_failure( - key=f"mode-query-{report_token}", - reason=f"Unable to retrieve queries for report token: {report_token}, " - f"Reason: {str(http_error)}", + title="Failed to Retrieve Queries", + message="Unable to retrieve queries for report token.", + context=f"Report Token: {report_token}, Error: {str(http_error)}", ) return queries @@ -1354,9 +1358,9 @@ def _get_last_query_run( queries = queries_json.get("_embedded", {}).get("queries", {}) except HTTPError as http_error: self.report.report_failure( - key=f"mode-query-{report_token}", - reason=f"Unable to retrieve queries for report token: {report_token}, " - f"Reason: {str(http_error)}", + title="Failed to Retrieve Queries for Report", + message="Unable to retrieve queries for report token.", + context=f"Report Token:{report_token}, Error: {str(http_error)}", ) return {} return queries @@ -1372,11 +1376,11 @@ def _get_charts(self, report_token: str, query_token: str) -> list: charts = charts_json.get("_embedded", {}).get("charts", {}) except HTTPError as http_error: self.report.report_failure( - key=f"mode-chart-{report_token}-{query_token}", - reason=f"Unable to retrieve charts: " - f"Report token: {report_token} " + title="Failed to Retrieve Charts", + message="Unable to retrieve charts from Mode.", + context=f"Report Token: {report_token}, " f"Query token: {query_token}, " - f"Reason: {str(http_error)}", + f"Error: {str(http_error)}", ) return charts diff --git a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py index af6b44677dffa..7ce3b5bc34da2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py +++ b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py @@ -317,8 +317,9 @@ def get_pymongo_type_string( try: type_string = PYMONGO_TYPE_TO_MONGO_TYPE[field_type] except KeyError: - self.report.report_warning( - collection_name, f"unable to map type {field_type} to metadata schema" + self.report.warning( + message="Unrecognized column types found", + context=f"Collection: {collection_name}, field type {field_type}", ) PYMONGO_TYPE_TO_MONGO_TYPE[field_type] = "unknown" type_string = "unknown" @@ -341,8 +342,9 @@ def get_field_type( TypeClass: Optional[Type] = _field_type_mapping.get(field_type) if TypeClass is None: - self.report.report_warning( - collection_name, f"unable to map type {field_type} to metadata schema" + self.report.warning( + message="Unrecognized column type found", + context=f"Collection: {collection_name}, field type {field_type}", ) TypeClass = NullTypeClass @@ -418,8 +420,9 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: if collection_schema_size > max_schema_size: # downsample the schema, using frequency as the sort key self.report.report_warning( - key=dataset_urn, - reason=f"Downsampling the collection schema because it has {collection_schema_size} fields. Threshold is {max_schema_size}", + title="Too many schema fields", + message=f"Downsampling the collection schema because it has too many schema fields. Configured threshold is {max_schema_size}", + context=f"Schema Size: {collection_schema_size}, Collection: {dataset_urn}", ) # Add this information to the custom properties so user can know they are looking at downsampled schema dataset_properties.customProperties[ diff --git a/metadata-ingestion/src/datahub/ingestion/source/nifi.py b/metadata-ingestion/src/datahub/ingestion/source/nifi.py index d650a9b1d1b4c..1ad55407e2750 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/nifi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/nifi.py @@ -570,9 +570,9 @@ def update_flow(self, pg_flow_dto: Dict) -> None: # noqa: C901 ) if not pg_response.ok: - self.report_warning( - self.config.site_url, + self.report.warning( "Failed to get process group flow " + pg.get("id"), + self.config.site_url, ) continue @@ -619,11 +619,11 @@ def update_flow_keep_only_ingress_egress(self): or c.name.startswith("Fetch") or c.name.startswith("Put") ): - self.report_warning( - self.config.site_url, + self.report.warning( f"Dropping NiFi Processor of type {c.type}, id {c.id}, name {c.name} from lineage view. \ This is likely an Ingress or Egress node which may be reading to/writing from external datasets \ However not currently supported in datahub", + self.config.site_url, ) else: logger.debug( @@ -734,10 +734,10 @@ def fetch_provenance_events( processor, eventType, startDate, oldest_event_time ) else: - self.report_warning( - self.config.site_url, - f"provenance events could not be fetched for processor \ + self.report.warning( + f"Provenance events could not be fetched for processor \ {processor.id} of type {processor.name}", + self.config.site_url, ) logger.warning(provenance_response.text) return @@ -794,10 +794,6 @@ def submit_provenance_query(self, processor, eventType, startDate, endDate): return provenance_response - def report_warning(self, key: str, reason: str) -> None: - logger.warning(f"{key}: {reason}") - self.report.report_warning(key, reason) - def delete_provenance(self, provenance_uri): delete_response = self.session.delete(provenance_uri) if not delete_response.ok: @@ -826,7 +822,7 @@ def construct_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 outgoing = list( filter(lambda x: x[0] == component.id, self.nifi_flow.connections) ) - inputJobs = [] + inputJobs = set() jobProperties = None if component.nifi_type is NifiType.PROCESSOR: @@ -877,7 +873,7 @@ def construct_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 dataset_urn, ) else: - inputJobs.append( + inputJobs.add( builder.make_data_job_urn_with_flow(flow_urn, incoming_from) ) @@ -900,11 +896,11 @@ def construct_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 site_urls = component.target_uris.split(",") # type: ignore for site_url in site_urls: if site_url not in self.config.site_url_to_site_name: - self.report_warning( - site_url, + self.report.warning( f"Site with url {site_url} is being used in flow but\ corresponding site name is not configured via site_url_to_site_name.\ This may result in broken lineage.", + site_url, ) else: site_name = self.config.site_url_to_site_name[site_url] @@ -921,11 +917,11 @@ def construct_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 site_urls = component.target_uris.split(",") # type: ignore for site_url in site_urls: if site_url not in self.config.site_url_to_site_name: - self.report_warning( - self.config.site_url, + self.report.warning( f"Site with url {site_url} is being used in flow but\ corresponding site name is not configured via site_url_to_site_name.\ This may result in broken lineage.", + self.config.site_url, ) else: site_name = self.config.site_url_to_site_name[site_url] @@ -957,7 +953,7 @@ def construct_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 job_properties=jobProperties, inlets=list(component.inlets.keys()), outlets=list(component.outlets.keys()), - inputJobs=inputJobs, + inputJobs=list(inputJobs), status=component.status, ) @@ -1054,17 +1050,15 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: try: self.authenticate() except Exception as e: - logger.error("Failed to authenticate", exc_info=e) - self.report.report_failure(self.config.site_url, "Failed to authenticate") + self.report.failure("Failed to authenticate", self.config.site_url, exc=e) return # Creates nifi_flow by invoking /flow rest api and saves as self.nifi_flow try: self.create_nifi_flow() except Exception as e: - logger.error("Failed to get root process group flow", exc_info=e) - self.report.report_failure( - self.config.site_url, "Failed to get root process group flow" + self.report.failure( + "Failed to get root process group flow", self.config.site_url, exc=e ) return diff --git a/metadata-ingestion/src/datahub/ingestion/source/openapi.py b/metadata-ingestion/src/datahub/ingestion/source/openapi.py index 0aa524547bd8e..8289265483d59 100755 --- a/metadata-ingestion/src/datahub/ingestion/source/openapi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/openapi.py @@ -184,27 +184,40 @@ def __init__(self, config: OpenApiConfig, ctx: PipelineContext, platform: str): self.report = SourceReport() self.url_basepath = "" - def report_bad_responses(self, status_code: int, key: str) -> None: + def report_bad_responses(self, status_code: int, type: str) -> None: if status_code == 400: self.report.report_warning( - key=key, reason="Unknown error for reaching endpoint" + title="Failed to Extract Metadata", + message="Bad request body when retrieving data from OpenAPI endpoint", + context=f"Endpoint Type: {type}, Status Code: {status_code}", ) elif status_code == 403: - self.report.report_warning(key=key, reason="Not authorised to get endpoint") + self.report.report_warning( + title="Unauthorized to Extract Metadata", + message="Received unauthorized response when attempting to retrieve data from OpenAPI endpoint", + context=f"Endpoint Type: {type}, Status Code: {status_code}", + ) elif status_code == 404: self.report.report_warning( - key=key, - reason="Unable to find an example for endpoint. Please add it to the list of forced examples.", + title="Failed to Extract Metadata", + message="Unable to find an example for endpoint. Please add it to the list of forced examples.", + context=f"Endpoint Type: {type}, Status Code: {status_code}", ) elif status_code == 500: self.report.report_warning( - key=key, reason="Server error for reaching endpoint" + title="Failed to Extract Metadata", + message="Received unknown server error from OpenAPI endpoint", + context=f"Endpoint Type: {type}, Status Code: {status_code}", ) elif status_code == 504: - self.report.report_warning(key=key, reason="Timeout for reaching endpoint") + self.report.report_warning( + title="Failed to Extract Metadata", + message="Timed out when attempting to retrieve data from OpenAPI endpoint", + context=f"Endpoint Type: {type}, Status Code: {status_code}", + ) else: raise Exception( - f"Unable to retrieve endpoint, response code {status_code}, key {key}" + f"Unable to retrieve endpoint, response code {status_code}, key {type}" ) def init_dataset( @@ -271,7 +284,7 @@ def get_workunits_internal(self) -> Iterable[ApiWorkUnit]: # noqa: C901 for w in warn_c: w_msg = w.message w_spl = w_msg.args[0].split(" --- ") # type: ignore - self.report.report_warning(key=w_spl[1], reason=w_spl[0]) + self.report.report_warning(message=w_spl[1], context=w_spl[0]) # here we put a sample from the "listing endpoint". To be used for later guessing of comosed endpoints. root_dataset_samples = {} @@ -293,8 +306,9 @@ def get_workunits_internal(self) -> Iterable[ApiWorkUnit]: # noqa: C901 yield self.build_wu(dataset_snapshot, dataset_name) elif endpoint_dets["method"] != "get": self.report.report_warning( - key=endpoint_k, - reason=f"No example provided for {endpoint_dets['method']}", + title="Failed to Extract Endpoint Metadata", + message=f"No example provided for {endpoint_dets['method']}", + context=f"Endpoint Type: {endpoint_k}, Name: {dataset_name}", ) continue # Only test endpoints if they're GETs elif ( @@ -319,13 +333,16 @@ def get_workunits_internal(self) -> Iterable[ApiWorkUnit]: # noqa: C901 response, dataset_name ) if not fields2add: - self.report.report_warning(key=endpoint_k, reason="No Fields") + self.report.info( + message="No fields found from endpoint response.", + context=f"Endpoint Type: {endpoint_k}, Name: {dataset_name}", + ) schema_metadata = set_metadata(dataset_name, fields2add) dataset_snapshot.aspects.append(schema_metadata) yield self.build_wu(dataset_snapshot, dataset_name) else: - self.report_bad_responses(response.status_code, key=endpoint_k) + self.report_bad_responses(response.status_code, type=endpoint_k) else: if endpoint_k not in config.forced_examples.keys(): # start guessing... @@ -347,15 +364,16 @@ def get_workunits_internal(self) -> Iterable[ApiWorkUnit]: # noqa: C901 if response.status_code == 200: fields2add, _ = extract_fields(response, dataset_name) if not fields2add: - self.report.report_warning( - key=endpoint_k, reason="No Fields" + self.report.info( + message="No fields found from endpoint response.", + context=f"Endpoint Type: {endpoint_k}, Name: {dataset_name}", ) schema_metadata = set_metadata(dataset_name, fields2add) dataset_snapshot.aspects.append(schema_metadata) yield self.build_wu(dataset_snapshot, dataset_name) else: - self.report_bad_responses(response.status_code, key=endpoint_k) + self.report_bad_responses(response.status_code, type=endpoint_k) else: composed_url = compose_url_attr( raw_url=endpoint_k, attr_list=config.forced_examples[endpoint_k] @@ -377,15 +395,16 @@ def get_workunits_internal(self) -> Iterable[ApiWorkUnit]: # noqa: C901 if response.status_code == 200: fields2add, _ = extract_fields(response, dataset_name) if not fields2add: - self.report.report_warning( - key=endpoint_k, reason="No Fields" + self.report.info( + message="No fields found from endpoint response.", + context=f"Endpoint Type: {endpoint_k}, Name: {dataset_name}", ) schema_metadata = set_metadata(dataset_name, fields2add) dataset_snapshot.aspects.append(schema_metadata) yield self.build_wu(dataset_snapshot, dataset_name) else: - self.report_bad_responses(response.status_code, key=endpoint_k) + self.report_bad_responses(response.status_code, type=endpoint_k) def get_report(self): return self.report diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py index 6e74bfda3743c..bd80433bc2e6c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py @@ -46,6 +46,7 @@ class Constant: Authorization = "Authorization" WORKSPACE_ID = "workspaceId" DASHBOARD_ID = "powerbi.linkedin.com/dashboards/{}" + DATASET_EXECUTE_QUERIES = "DATASET_EXECUTE_QUERIES_POST" DATASET_ID = "datasetId" REPORT_ID = "reportId" SCAN_ID = "ScanId" @@ -59,9 +60,12 @@ class Constant: STATUS = "status" CHART_ID = "powerbi.linkedin.com/charts/{}" CHART_KEY = "chartKey" + COLUMN_TYPE = "columnType" + DATA_TYPE = "dataType" DASHBOARD = "dashboard" DASHBOARDS = "dashboards" DASHBOARD_KEY = "dashboardKey" + DESCRIPTION = "description" OWNERSHIP = "ownership" BROWSERPATH = "browsePaths" DASHBOARD_INFO = "dashboardInfo" @@ -108,6 +112,7 @@ class Constant: TABLES = "tables" EXPRESSION = "expression" SOURCE = "source" + SCHEMA_METADATA = "schemaMetadata" PLATFORM_NAME = "powerbi" REPORT_TYPE_NAME = BIAssetSubTypes.REPORT CHART_COUNT = "chartCount" @@ -238,6 +243,13 @@ class OwnershipMapping(ConfigModel): ) +class PowerBiProfilingConfig(ConfigModel): + enabled: bool = pydantic.Field( + default=False, + description="Whether profiling of PowerBI datasets should be done", + ) + + class PowerBiDashboardSourceConfig( StatefulIngestionConfigBase, DatasetSourceConfigMixin ): @@ -421,6 +433,13 @@ class PowerBiDashboardSourceConfig( "Works for M-Query where native SQL is used for transformation.", ) + profile_pattern: AllowDenyPattern = pydantic.Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns to filter tables for profiling during ingestion. Note that only tables " + "allowed by the `table_pattern` will be considered. Matched format is 'workspacename.datasetname.tablename'", + ) + profiling: PowerBiProfilingConfig = PowerBiProfilingConfig() + @root_validator(skip_on_failure=True) def validate_extract_column_level_lineage(cls, values: Dict) -> Dict: flags = [ diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py index 16f174525254d..de4eaf6b64434 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py @@ -67,7 +67,9 @@ CorpUserKeyClass, DashboardInfoClass, DashboardKeyClass, + DatasetFieldProfileClass, DatasetLineageTypeClass, + DatasetProfileClass, DatasetPropertiesClass, GlobalTagsClass, OtherSchemaClass, @@ -483,9 +485,64 @@ def to_datahub_dataset( Constant.DATASET, dataset.tags, ) + self.extract_profile(dataset_mcps, workspace, dataset, table, ds_urn) return dataset_mcps + def extract_profile( + self, + dataset_mcps: List[MetadataChangeProposalWrapper], + workspace: powerbi_data_classes.Workspace, + dataset: powerbi_data_classes.PowerBIDataset, + table: powerbi_data_classes.Table, + ds_urn: str, + ) -> None: + if not self.__config.profiling.enabled: + # Profiling not enabled + return + + if not self.__config.profile_pattern.allowed( + f"{workspace.name}.{dataset.name}.{table.name}" + ): + logger.info( + f"Table {table.name} in {dataset.name}, not allowed for profiling" + ) + return + logger.debug(f"Profiling table: {table.name}") + + profile = DatasetProfileClass(timestampMillis=builder.get_sys_time()) + profile.rowCount = table.row_count + profile.fieldProfiles = [] + + columns: List[ + Union[powerbi_data_classes.Column, powerbi_data_classes.Measure] + ] = [*(table.columns or []), *(table.measures or [])] + for column in columns: + allowed_column = self.__config.profile_pattern.allowed( + f"{workspace.name}.{dataset.name}.{table.name}.{column.name}" + ) + if column.isHidden or not allowed_column: + logger.info(f"Column {column.name} not allowed for profiling") + continue + measure_profile = column.measure_profile + if measure_profile: + field_profile = DatasetFieldProfileClass(column.name or "") + field_profile.sampleValues = measure_profile.sample_values + field_profile.min = measure_profile.min + field_profile.max = measure_profile.max + field_profile.uniqueCount = measure_profile.unique_count + profile.fieldProfiles.append(field_profile) + + profile.columnCount = table.column_count + + mcp = MetadataChangeProposalWrapper( + entityType="dataset", + entityUrn=ds_urn, + aspectName="datasetProfile", + aspect=profile, + ) + dataset_mcps.append(mcp) + @staticmethod def transform_tags(tags: List[str]) -> GlobalTagsClass: return GlobalTagsClass( @@ -1180,6 +1237,10 @@ def report_to_datahub_work_units( SourceCapability.LINEAGE_FINE, "Disabled by default, configured using `extract_column_level_lineage`. ", ) +@capability( + SourceCapability.DATA_PROFILING, + "Optionally enabled via configuration profiling.enabled", +) class PowerBiDashboardSource(StatefulIngestionSourceBase, TestableSource): """ This plugin extracts the following: diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py index ce4dd9a7a0c0f..6e8d939325d5b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py @@ -85,6 +85,14 @@ def __hash__(self): return hash(self.__members()) +@dataclass +class MeasureProfile: + min: Optional[str] = None + max: Optional[str] = None + unique_count: Optional[int] = None + sample_values: Optional[List[str]] = None + + @dataclass class Column: name: str @@ -96,6 +104,7 @@ class Column: columnType: Optional[str] = None expression: Optional[str] = None description: Optional[str] = None + measure_profile: Optional[MeasureProfile] = None @dataclass @@ -108,6 +117,7 @@ class Measure: BooleanTypeClass, DateTypeClass, NullTypeClass, NumberTypeClass, StringTypeClass ] = dataclasses.field(default_factory=NullTypeClass) description: Optional[str] = None + measure_profile: Optional[MeasureProfile] = None @dataclass @@ -117,6 +127,8 @@ class Table: expression: Optional[str] = None columns: Optional[List[Column]] = None measures: Optional[List[Measure]] = None + row_count: Optional[int] = None + column_count: Optional[int] = None # Pointer to the parent dataset. dataset: Optional["PowerBIDataset"] = None diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py index fadd7a48b62f7..b190cf065b6e3 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod from datetime import datetime, timedelta from time import sleep -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union import msal import requests @@ -11,18 +11,27 @@ from requests.adapters import HTTPAdapter from urllib3 import Retry -from datahub.configuration.common import ConfigurationError +from datahub.configuration.common import AllowDenyPattern, ConfigurationError from datahub.ingestion.source.powerbi.config import Constant from datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes import ( + Column, Dashboard, + Measure, + MeasureProfile, Page, PowerBIDataset, Report, + Table, Tile, User, Workspace, new_powerbi_dataset, ) +from datahub.ingestion.source.powerbi.rest_api_wrapper.profiling_utils import ( + process_column_result, + process_sample_result, +) +from datahub.ingestion.source.powerbi.rest_api_wrapper.query import DaxQuery # Logger instance logger = logging.getLogger(__name__) @@ -107,6 +116,16 @@ def get_tiles_endpoint(self, workspace: Workspace, dashboard_id: str) -> str: def _get_pages_by_report(self, workspace: Workspace, report_id: str) -> List[Page]: pass + @abstractmethod + def profile_dataset( + self, + dataset: PowerBIDataset, + table: Table, + workspace_name: str, + profile_pattern: Optional[AllowDenyPattern], + ) -> None: + pass + @abstractmethod def get_dataset( self, workspace_id: str, dataset_id: str @@ -387,6 +406,7 @@ class RegularAPIResolver(DataResolverBase): Constant.REPORT_GET: "{POWERBI_BASE_URL}/{WORKSPACE_ID}/reports/{REPORT_ID}", Constant.REPORT_LIST: "{POWERBI_BASE_URL}/{WORKSPACE_ID}/reports", Constant.PAGE_BY_REPORT: "{POWERBI_BASE_URL}/{WORKSPACE_ID}/reports/{REPORT_ID}/pages", + Constant.DATASET_EXECUTE_QUERIES: "{POWERBI_BASE_URL}/{WORKSPACE_ID}/datasets/{DATASET_ID}/executeQueries", } def get_dataset( @@ -515,6 +535,134 @@ def _get_pages_by_report(self, workspace: Workspace, report_id: str) -> List[Pag def get_users(self, workspace_id: str, entity: str, entity_id: str) -> List[User]: return [] # User list is not available in regular access + def _execute_profiling_query(self, dataset: PowerBIDataset, query: str) -> dict: + dataset_query_endpoint: str = self.API_ENDPOINTS[ + Constant.DATASET_EXECUTE_QUERIES + ] + # Replace place holders + dataset_query_endpoint = dataset_query_endpoint.format( + POWERBI_BASE_URL=self.BASE_URL, + WORKSPACE_ID=dataset.workspace_id, + DATASET_ID=dataset.id, + ) + # Hit PowerBi + logger.info(f"Request to query endpoint URL={dataset_query_endpoint}") + + # Serializer is configured to include nulls so that the queried fields + # exist in the returned payloads. Only failed queries will result in KeyError + payload = { + "queries": [ + { + "query": query, + } + ], + "serializerSettings": { + "includeNulls": True, + }, + } + response = self._request_session.post( + dataset_query_endpoint, + json=payload, + headers=self.get_authorization_header(), + ) + response.raise_for_status() + return response.json() + + def _get_row_count(self, dataset: PowerBIDataset, table: Table) -> int: + query = DaxQuery.row_count_query(table.name) + try: + data = self._execute_profiling_query(dataset, query) + rows = data["results"][0]["tables"][0]["rows"] + count = rows[0]["[count]"] + return count + except requests.exceptions.RequestException as ex: + logger.warning(getattr(ex.response, "text", "")) + logger.warning( + f"Profiling failed for getting row count for dataset {dataset.id}, with status code {getattr(ex.response, 'status_code', None)}", + ) + except (KeyError, IndexError) as ex: + logger.warning( + f"Profiling failed for getting row count for dataset {dataset.id}, with {ex}" + ) + return 0 + + def _get_data_sample(self, dataset: PowerBIDataset, table: Table) -> dict: + try: + query = DaxQuery.data_sample_query(table.name) + data = self._execute_profiling_query(dataset, query) + return process_sample_result(data) + except requests.exceptions.RequestException as ex: + logger.warning(getattr(ex.response, "text", "")) + logger.warning( + f"Getting sample with TopN failed for dataset {dataset.id}, with status code {getattr(ex.response, 'status_code', None)}", + ) + except (KeyError, IndexError) as ex: + logger.warning( + f"Getting sample with TopN failed for dataset {dataset.id}, with {ex}" + ) + return {} + + def _get_column_data( + self, dataset: PowerBIDataset, table: Table, column: Union[Column, Measure] + ) -> dict: + try: + logger.debug(f"Column data query for {dataset.name}, {column.name}") + query = DaxQuery.column_data_query(table.name, column.name) + data = self._execute_profiling_query(dataset, query) + return process_column_result(data) + except requests.exceptions.RequestException as ex: + logger.warning(getattr(ex.response, "text", "")) + logger.warning( + f"Getting column statistics failed for dataset {dataset.name}, {column.name}, with status code {getattr(ex.response, 'status_code', None)}", + ) + except (KeyError, IndexError) as ex: + logger.warning( + f"Getting column statistics failed for dataset {dataset.name}, {column.name}, with {ex}" + ) + return {} + + def profile_dataset( + self, + dataset: PowerBIDataset, + table: Table, + workspace_name: str, + profile_pattern: Optional[AllowDenyPattern], + ) -> None: + if not profile_pattern: + logger.info("Profile pattern not configured, not profiling") + return + + if not profile_pattern.allowed(f"{workspace_name}.{dataset.name}.{table.name}"): + logger.info( + f"Table {table.name} in {dataset.name}, not allowed for profiling" + ) + return + + logger.info(f"Profiling table: {table.name}") + row_count = self._get_row_count(dataset, table) + sample = self._get_data_sample(dataset, table) + + table.row_count = row_count + column_count = 0 + + columns: List[Union[Column, Measure]] = [ + *(table.columns or []), + *(table.measures or []), + ] + for column in columns: + if column.isHidden: + continue + + column_sample = sample.get(column.name, None) if sample else None + column_stats = self._get_column_data(dataset, table, column) + + column.measure_profile = MeasureProfile( + sample_values=column_sample, **column_stats + ) + column_count += 1 + + table.column_count = column_count + class AdminAPIResolver(DataResolverBase): # Admin access endpoints @@ -817,3 +965,13 @@ def get_dataset_parameters( ) -> Dict[str, str]: logger.debug("Get dataset parameter is unsupported in Admin API") return {} + + def profile_dataset( + self, + dataset: PowerBIDataset, + table: Table, + workspace_name: str, + profile_pattern: Optional[AllowDenyPattern], + ) -> None: + logger.debug("Profile dataset is unsupported in Admin API") + return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py index d72624bd70512..7ce0dcb2e3cf4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py @@ -287,11 +287,12 @@ def _parse_endorsement(endorsements: Optional[dict]) -> List[str]: return [endorsement] - def _get_workspace_datasets(self, scan_result: Optional[dict]) -> dict: + def _get_workspace_datasets(self, workspace: Workspace) -> dict: """ Filter out "dataset" from scan_result and return Dataset instance set """ dataset_map: dict = {} + scan_result = workspace.scan_result if scan_result is None: return dataset_map @@ -345,30 +346,37 @@ def _get_workspace_datasets(self, scan_result: Optional[dict]) -> dict: and len(table[Constant.SOURCE]) > 0 else None ) - dataset_instance.tables.append( - Table( - name=table[Constant.NAME], - full_name="{}.{}".format( - dataset_name.replace(" ", "_"), - table[Constant.NAME].replace(" ", "_"), - ), - expression=expression, - columns=[ - Column( - **column, - datahubDataType=FIELD_TYPE_MAPPING.get( - column["dataType"], FIELD_TYPE_MAPPING["Null"] - ), - ) - for column in table.get("columns", []) - ], - measures=[ - Measure(**measure) for measure in table.get("measures", []) - ], - dataset=dataset_instance, - ) + table = Table( + name=table[Constant.NAME], + full_name="{}.{}".format( + dataset_name.replace(" ", "_"), + table[Constant.NAME].replace(" ", "_"), + ), + expression=expression, + columns=[ + Column( + **column, + datahubDataType=FIELD_TYPE_MAPPING.get( + column["dataType"], FIELD_TYPE_MAPPING["Null"] + ), + ) + for column in table.get("columns", []) + ], + measures=[ + Measure(**measure) for measure in table.get("measures", []) + ], + dataset=dataset_instance, + row_count=None, + column_count=None, ) - + if self.__config.profiling.enabled: + self._get_resolver().profile_dataset( + dataset_instance, + table, + workspace.name, + self.__config.profile_pattern, + ) + dataset_instance.tables.append(table) return dataset_map def _fill_metadata_from_scan_result( @@ -393,9 +401,7 @@ def _fill_metadata_from_scan_result( independent_datasets=[], ) cur_workspace.scan_result = workspace_metadata - cur_workspace.datasets = self._get_workspace_datasets( - cur_workspace.scan_result - ) + cur_workspace.datasets = self._get_workspace_datasets(cur_workspace) # Fetch endorsements tag if it is enabled from configuration if self.__config.extract_endorsements_to_tags: diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/profiling_utils.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/profiling_utils.py new file mode 100644 index 0000000000000..35e4cea41264a --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/profiling_utils.py @@ -0,0 +1,46 @@ +import re +from typing import Dict, List, Optional + + +def get_column_name(table_and_col: str) -> Optional[str]: + regex = re.compile(".*\\[(.*)\\]$") + m = regex.match(table_and_col) + if m: + return m.group(1) + return None + + +def process_sample_result(result_data: dict) -> dict: + sample_data_by_column: Dict[str, List[str]] = {} + rows = result_data["results"][0]["tables"][0]["rows"] + for sample in rows: + for key, value in sample.items(): + if not value: + continue + column_name = get_column_name(key) + + if not column_name: + continue + + if column_name not in sample_data_by_column: + sample_data_by_column[column_name] = [] + sample_data_by_column[column_name].append(str(value)) + return sample_data_by_column + + +def process_column_result(result_data: dict) -> dict: + sample_data_by_column: Dict[str, str] = {} + rows = result_data["results"][0]["tables"][0]["rows"] + for sample in rows: + for key, value in sample.items(): + if not value: + continue + column_name = get_column_name(key) + + if not column_name: + continue + + if column_name != "unique_count": + value = str(value) + sample_data_by_column[column_name] = value + return sample_data_by_column diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/query.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/query.py new file mode 100644 index 0000000000000..cb66210efc8e5 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/query.py @@ -0,0 +1,17 @@ +class DaxQuery: + @staticmethod + def data_sample_query(table_name: str) -> str: + return f"EVALUATE TOPN(3, '{table_name}')" + + @staticmethod + def column_data_query(table_name: str, column_name: str) -> str: + return f""" + EVALUATE ROW( + "min", MIN('{table_name}'[{column_name}]), + "max", MAX('{table_name}'[{column_name}]), + "unique_count", COUNTROWS ( DISTINCT ( '{table_name}'[{column_name}] ) ) + )""" + + @staticmethod + def row_count_query(table_name: str) -> str: + return f"""EVALUATE ROW("count", COUNTROWS ( '{table_name}' ))""" diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py index eb0b9d02d865e..b5547e8a8ae9e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py @@ -93,6 +93,8 @@ class Constant: # Personal entity constants PERSONAL_SPACE_ID = "personal-space-id" PERSONAL_SPACE_NAME = "personal_space" + # Hypercube + HYPERCUBE = "qHyperCube" @dataclass diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py index c30b456253e06..672fcbceb0603 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py @@ -164,8 +164,8 @@ class Chart(BaseModel): @root_validator(pre=True) def update_values(cls, values: Dict) -> Dict: values[Constant.QID] = values[Constant.QINFO][Constant.QID] - values["qDimension"] = values["qHyperCube"]["qDimensionInfo"] - values["qMeasure"] = values["qHyperCube"]["qMeasureInfo"] + values["qDimension"] = values[Constant.HYPERCUBE]["qDimensionInfo"] + values["qMeasure"] = values[Constant.HYPERCUBE]["qMeasureInfo"] return values diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py index d7a040ff5f0a0..31b0731aaa751 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py @@ -113,7 +113,13 @@ def _get_chart( method="GetChild", params={"qId": chart_id} ) response = websocket_connection.websocket_send_request(method="GetLayout") - return Chart.parse_obj(response[Constant.QLAYOUT]) + q_layout = response[Constant.QLAYOUT] + if Constant.HYPERCUBE not in q_layout: + logger.warning( + f"Chart with id {chart_id} of sheet {sheet_id} does not have hypercube. q_layout: {q_layout}" + ) + return None + return Chart.parse_obj(q_layout) except Exception as e: self._log_http_error( message=f"Unable to fetch chart {chart_id} of sheet {sheet_id}. Exception: {e}" @@ -135,6 +141,11 @@ def _get_sheet( # That means sheet is private sheet return None sheet = Sheet.parse_obj(sheet_dict[Constant.QMETA]) + if Constant.QCHILDLIST not in sheet_dict: + logger.warning( + f"Sheet {sheet.title} with id {sheet_id} does not have any charts. sheet_dict: {sheet_dict}" + ) + return sheet for i, chart_dict in enumerate( sheet_dict[Constant.QCHILDLIST][Constant.QITEMS] ): diff --git a/metadata-ingestion/src/datahub/ingestion/source/redash.py b/metadata-ingestion/src/datahub/ingestion/source/redash.py index f7b8bb09724a1..c7a3f25e947dc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redash.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redash.py @@ -10,7 +10,7 @@ from pydantic.fields import Field from redash_toolbelt import Redash from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry +from urllib3.util.retry import Retry import datahub.emitter.mce_builder as builder from datahub.configuration.common import AllowDenyPattern, ConfigModel @@ -555,7 +555,7 @@ def _get_dashboard_snapshot(self, dashboard_data, redash_version): title = dashboard_data.get("name", "") last_modified = ChangeAuditStamps( - created=AuditStamp(time=modified_ts, actor=modified_actor), + created=None, lastModified=AuditStamp(time=modified_ts, actor=modified_actor), ) @@ -659,7 +659,7 @@ def _get_chart_type_from_viz_data(self, viz_data: Dict) -> str: viz_type = viz_data.get("type", "") viz_options = viz_data.get("options", {}) globalSeriesType = viz_options.get("globalSeriesType", "") - report_key = f"redash-chart-{viz_data['id']}" + report_type = f"redash-chart-{viz_data['id']}" # handle Plotly chart types if viz_type == "CHART": @@ -667,14 +667,14 @@ def _get_chart_type_from_viz_data(self, viz_data: Dict) -> str: if chart_type is None: chart_type = DEFAULT_VISUALIZATION_TYPE message = f"ChartTypeClass for Redash Visualization Type={viz_type} with options.globalSeriesType={globalSeriesType} is missing. Setting to {DEFAULT_VISUALIZATION_TYPE}" - self.report.report_warning(key=report_key, reason=message) + self.report.report_warning(title=report_type, message=message) logger.warning(message) else: chart_type = VISUALIZATION_TYPE_MAP.get(viz_type) if chart_type is None: chart_type = DEFAULT_VISUALIZATION_TYPE message = f"ChartTypeClass for Redash Visualization Type={viz_type} is missing. Setting to {DEFAULT_VISUALIZATION_TYPE}" - self.report.report_warning(key=report_key, reason=message) + self.report.report_warning(title=report_type, message=message) logger.warning(message) return chart_type @@ -694,7 +694,7 @@ def _get_chart_snapshot(self, query_data: Dict, viz_data: Dict) -> ChartSnapshot title = f"{query_data.get('name')} {viz_data.get('name', '')}" last_modified = ChangeAuditStamps( - created=AuditStamp(time=modified_ts, actor=modified_actor), + created=None, lastModified=AuditStamp(time=modified_ts, actor=modified_actor), ) @@ -713,8 +713,8 @@ def _get_chart_snapshot(self, query_data: Dict, viz_data: Dict) -> ChartSnapshot self.report.charts_no_input.add(chart_urn) self.report.queries_no_dataset.add(str(query_id)) self.report.report_warning( - key="redash-chart-input-missing", - reason=f"For viz-id-{viz_id}-query-{query_id}-datasource-{data_source_id} data_source_type={data_source_type} no datasources found. Setting inputs to None", + title="redash-chart-input-missing", + message=f"For viz-id-{viz_id}-query-{query_id}-datasource-{data_source_id} data_source_type={data_source_type} no datasources found. Setting inputs to None", ) chart_info = ChartInfoClass( diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py index 74507d850014a..2ff73323a14e3 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py @@ -49,6 +49,11 @@ class S3LineageProviderConfig(ConfigModel): description="Strip filename from s3 url. It only applies if path_specs are not specified.", ) + ignore_non_path_spec_path: bool = Field( + default=False, + description="Ignore paths that are not match in path_specs. It only applies if path_specs are specified.", + ) + class S3DatasetLineageProviderConfigBase(ConfigModel): """ diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/exception.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/exception.py new file mode 100644 index 0000000000000..43ad5bfcefdf1 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/exception.py @@ -0,0 +1,65 @@ +from typing import Callable, Iterable, TypeVar, Union + +import redshift_connector +from typing_extensions import ParamSpec + +from datahub.ingestion.source.redshift.report import RedshiftReport + +T = TypeVar("T") +P = ParamSpec("P") + + +def handle_redshift_exceptions( + report: RedshiftReport, + func: Callable[P, T], + *args: P.args, + **kwargs: P.kwargs, +) -> Union[T, None]: + try: + return func(*args, **kwargs) + except redshift_connector.Error as e: + report_redshift_failure(report, e) + return None + + +def handle_redshift_exceptions_yield( + report: RedshiftReport, + func: Callable[P, Iterable[T]], + *args: P.args, + **kwargs: P.kwargs, +) -> Iterable[T]: + try: + yield from func(*args, **kwargs) + except redshift_connector.Error as e: + report_redshift_failure(report, e) + + +def report_redshift_failure( + report: RedshiftReport, e: redshift_connector.Error +) -> None: + error_message = str(e).lower() + if "permission denied" in error_message: + if "svv_table_info" in error_message: + report.report_failure( + title="Permission denied", + message="Failed to extract metadata due to insufficient permission to access 'svv_table_info' table. Please ensure the provided database user has access.", + exc=e, + ) + elif "svl_user_info" in error_message: + report.report_failure( + title="Permission denied", + message="Failed to extract metadata due to insufficient permission to access 'svl_user_info' table. Please ensure the provided database user has access.", + exc=e, + ) + else: + report.report_failure( + title="Permission denied", + message="Failed to extract metadata due to insufficient permissions.", + exc=e, + ) + else: + report.report_failure( + title="Failed to extract some metadata", + message="Failed to extract some metadata from Redshift.", + exc=e, + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py index 6c6267e80ee62..dadb06b6a95e2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, field from datetime import datetime from enum import Enum -from typing import Dict, List, Optional, Set, Tuple, Union +from typing import Dict, Iterable, List, Optional, Set, Tuple, Union from urllib.parse import urlparse import humanfriendly @@ -53,6 +53,7 @@ ) from datahub.metadata.urns import DatasetUrn from datahub.sql_parsing.schema_resolver import SchemaResolver +from datahub.sql_parsing.sqlglot_utils import get_dialect, parse_statement from datahub.utilities import memory_footprint from datahub.utilities.dedup_list import deduplicate_list @@ -128,7 +129,7 @@ def parse_alter_table_rename(default_schema: str, query: str) -> Tuple[str, str, Parses an ALTER TABLE ... RENAME TO ... query and returns the schema, previous table name, and new table name. """ - parsed_query = sqlglot.parse_one(query, dialect="redshift") + parsed_query = parse_statement(query, dialect=get_dialect("redshift")) assert isinstance(parsed_query, sqlglot.exp.AlterTable) prev_name = parsed_query.this.name rename_clause = parsed_query.args["actions"][0] @@ -264,13 +265,23 @@ def warn(self, log: logging.Logger, key: str, reason: str) -> None: # TODO: Remove this method. self.report.warning(key, reason) - def _get_s3_path(self, path: str) -> str: + def _get_s3_path(self, path: str) -> Optional[str]: if self.config.s3_lineage_config: for path_spec in self.config.s3_lineage_config.path_specs: if path_spec.allowed(path): _, table_path = path_spec.extract_table_name_and_path(path) return table_path + if ( + self.config.s3_lineage_config.ignore_non_path_spec_path + and len(self.config.s3_lineage_config.path_specs) > 0 + ): + self.report.num_lineage_dropped_s3_path += 1 + logger.debug( + f"Skipping s3 path {path} as it does not match any path spec." + ) + return None + if self.config.s3_lineage_config.strip_urls: if "/" in urlparse(path).path: return str(path.rsplit("/", 1)[0]) @@ -323,13 +334,14 @@ def _get_sources_from_query( ), ) - def _build_s3_path_from_row(self, filename: str) -> str: + def _build_s3_path_from_row(self, filename: str) -> Optional[str]: path = filename.strip() if urlparse(path).scheme != "s3": raise ValueError( f"Only s3 source supported with copy/unload. The source was: {path}" ) - return strip_s3_prefix(self._get_s3_path(path)) + s3_path = self._get_s3_path(path) + return strip_s3_prefix(s3_path) if s3_path else None def _get_sources( self, @@ -369,7 +381,11 @@ def _get_sources( ) self.report.num_lineage_dropped_not_support_copy_path += 1 return [], None - path = strip_s3_prefix(self._get_s3_path(path)) + s3_path = self._get_s3_path(path) + if s3_path is None: + return [], None + + path = strip_s3_prefix(s3_path) urn = make_dataset_urn_with_platform_instance( platform=platform.value, name=path, @@ -539,6 +555,8 @@ def _get_target_lineage( target_platform = LineageDatasetPlatform.S3 # Following call requires 'filename' key in lineage_row target_path = self._build_s3_path_from_row(lineage_row.filename) + if target_path is None: + return None urn = make_dataset_urn_with_platform_instance( platform=target_platform.value, name=target_path, @@ -644,7 +662,7 @@ def populate_lineage( if self.config.resolve_temp_table_in_lineage: self._init_temp_table_schema( database=database, - temp_tables=self.get_temp_tables(connection=connection), + temp_tables=list(self.get_temp_tables(connection=connection)), ) populate_calls: List[Tuple[str, LineageCollectorType]] = [] @@ -848,10 +866,19 @@ def _process_table_renames( for rename_row in RedshiftDataDictionary.get_alter_table_commands( connection, query ): - schema, prev_name, new_name = parse_alter_table_rename( - default_schema=self.config.default_schema, - query=rename_row.query_text, - ) + # Redshift's system table has some issues where it encodes newlines as \n instead a proper + # newline character. This can cause issues in our parser. + query_text = rename_row.query_text.replace("\\n", "\n") + + try: + schema, prev_name, new_name = parse_alter_table_rename( + default_schema=self.config.default_schema, + query=query_text, + ) + except ValueError as e: + logger.info(f"Failed to parse alter table rename: {e}") + self.report.num_alter_table_parse_errors += 1 + continue prev_urn = make_dataset_urn_with_platform_instance( platform=LineageDatasetPlatform.REDSHIFT.value, @@ -876,7 +903,7 @@ def _process_table_renames( def get_temp_tables( self, connection: redshift_connector.Connection - ) -> List[TempTableRow]: + ) -> Iterable[TempTableRow]: ddl_query: str = self.queries.temp_table_ddl_query( start_time=self.config.start_time, end_time=self.config.end_time, @@ -884,15 +911,11 @@ def get_temp_tables( logger.debug(f"Temporary table ddl query = {ddl_query}") - temp_table_rows: List[TempTableRow] = [] - for row in RedshiftDataDictionary.get_temporary_rows( conn=connection, query=ddl_query, ): - temp_table_rows.append(row) - - return temp_table_rows + yield row def find_temp_tables( self, temp_table_rows: List[TempTableRow], temp_table_names: List[str] diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py index 2c7ebb613c57a..3d6c746183fd9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py @@ -1,6 +1,5 @@ import collections import logging -import traceback from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union import redshift_connector @@ -105,7 +104,15 @@ def build( for schema, tables in schemas.items() for table in tables } - self.aggregator.is_temp_table = lambda urn: urn not in self.known_urns + self.aggregator._is_temp_table = ( + lambda name: DatasetUrn.create_from_ids( + self.platform, + name, + env=self.config.env, + platform_instance=self.config.platform_instance, + ).urn() + not in self.known_urns + ) # Handle all the temp tables up front. if self.config.resolve_temp_table_in_lineage: @@ -116,7 +123,11 @@ def build( default_schema=self.config.default_schema, session_id=temp_row.session_id, query_timestamp=temp_row.start_time, - is_known_temp_table=True, + # The "temp table" query actually returns all CREATE TABLE statements, even if they + # aren't explicitly a temp table. As such, setting is_known_temp_table=True + # would not be correct. We already have mechanisms to autodetect temp tables, + # so we won't lose anything by not setting it. + is_known_temp_table=False, ) populate_calls: List[Tuple[LineageCollectorType, str, Callable]] = [] @@ -237,8 +248,10 @@ def _populate_lineage_agg( processor(lineage_row) except Exception as e: self.report.warning( - f"lineage-v2-extract-{lineage_type.name}", - f"Error was {e}, {traceback.format_exc()}", + title="Failed to extract some lineage", + message=f"Failed to extract lineage of type {lineage_type.name}", + context=f"Query: '{query}'", + exc=e, ) self._lineage_v1.report_status(f"extract-{lineage_type.name}", False) @@ -405,3 +418,9 @@ def _process_external_tables( def generate(self) -> Iterable[MetadataWorkUnit]: for mcp in self.aggregator.gen_metadata(): yield mcp.as_workunit() + if len(self.aggregator.report.observed_query_parse_failures) > 0: + self.report.report_failure( + title="Failed to extract some SQL lineage", + message="Unexpected error(s) while attempting to extract lineage from SQL queries. See the full logs for more details.", + context=f"Query Parsing Failures: {self.aggregator.report.observed_query_parse_failures}", + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/profile.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/profile.py index eed82ec4d83e7..6f611fa674187 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/profile.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/profile.py @@ -48,15 +48,26 @@ def get_workunits( if not self.config.schema_pattern.allowed(schema): continue for table in tables[db].get(schema, {}): - if ( - not self.config.profiling.profile_external_tables - and table.type == "EXTERNAL_TABLE" - ): - self.report.profiling_skipped_other[schema] += 1 - logger.info( - f"Skipping profiling of external table {db}.{schema}.{table.name}" - ) - continue + if table.type == "EXTERNAL_TABLE": + if not self.config.profiling.profile_external_tables: + # Case 1: If user did not tell us to profile external tables, simply log this. + self.report.profiling_skipped_other[schema] += 1 + logger.info( + f"Skipping profiling of external table {db}.{schema}.{table.name}" + ) + # Continue, since we should not profile this table. + continue + elif self.config.profiling.profile_table_level_only: + # Case 2: User DID tell us to profile external tables, but only at the table level. + # Currently, we do not support this combination. The user needs to also set + # profile_table_level_only to False in order to profile. + self.report.report_warning( + title="Skipped profiling for external tables", + message="External tables are not supported for profiling when 'profile_table_level_only' config is set to 'True'. Please set 'profile_table_level_only' to 'False' in order to profile external Redshift tables.", + context=f"External Table: {db}.{schema}.{table.name}", + ) + # Continue, since we were unable to retrieve cheap profiling stats from svv_table_info. + continue # Emit the profile work unit profile_request = self.get_profile_request(table, schema, db) if profile_request is not None: diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py index 1bc82556ce4bc..affbcd00b5107 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py @@ -122,6 +122,7 @@ def list_tables( else: return f"{tables_query} UNION {external_tables_query}" + # Why is this unused. Is this a bug? list_columns: str = """ SELECT n.nspname as "schema", @@ -502,7 +503,7 @@ def list_insert_create_queries_sql( usename as username, ddl, sq.query as query_id, - min(si.starttime) as starttime, + min(si.starttime) as timestamp, ANY_VALUE(pid) as session_id from stl_insert as si diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py index 6be7eedf976bd..a6ffed65aaa70 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py @@ -45,6 +45,7 @@ DatasetSubTypes, ) from datahub.ingestion.source.redshift.config import RedshiftConfig +from datahub.ingestion.source.redshift.exception import handle_redshift_exceptions_yield from datahub.ingestion.source.redshift.lineage import RedshiftLineageExtractor from datahub.ingestion.source.redshift.lineage_v2 import RedshiftSqlLineageV2 from datahub.ingestion.source.redshift.profile import RedshiftProfiler @@ -411,7 +412,12 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: ] def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]: - connection = RedshiftSource.get_redshift_connection(self.config) + connection = self._try_get_redshift_connection(self.config) + + if connection is None: + # If we failed to establish a connection, short circuit the connector. + return + database = self.config.database logger.info(f"Processing db {database}") self.report.report_ingestion_stage_start(METADATA_EXTRACTION) @@ -419,9 +425,20 @@ def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit self.db_views[database] = defaultdict() self.db_schemas.setdefault(database, {}) + # TODO: Ideally, we'd push down exception handling to the place where the connection is used, as opposed to keeping + # this fallback. For now, this gets us broad coverage quickly. + yield from handle_redshift_exceptions_yield( + self.report, self._extract_metadata, connection, database + ) + + def _extract_metadata( + self, connection: redshift_connector.Connection, database: str + ) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]: + yield from self.gen_database_container( database=database, ) + self.cache_tables_and_views(connection, database) self.report.tables_in_mem_size[database] = humanfriendly.format_size( @@ -556,6 +573,7 @@ def process_schema( ): for table in self.db_tables[schema.database][schema.name]: table.columns = schema_columns[schema.name].get(table.name, []) + table.column_count = len(table.columns) table_wu_generator = self._process_table( table, database=database ) @@ -575,8 +593,10 @@ def process_schema( f"Table processed: {schema.database}.{schema.name}.{table.name}" ) else: - logger.info( - f"No tables in cache for {schema.database}.{schema.name}, skipping" + self.report.info( + title="No tables found in some schemas", + message="No tables found in some schemas. This may be due to insufficient privileges for the provided user.", + context=f"Schema: {schema.database}.{schema.name}", ) else: logger.info("Table processing disabled, skipping") @@ -589,6 +609,7 @@ def process_schema( ): for view in self.db_views[schema.database][schema.name]: view.columns = schema_columns[schema.name].get(view.name, []) + view.column_count = len(view.columns) yield from self._process_view( table=view, database=database, schema=schema ) @@ -603,8 +624,10 @@ def process_schema( f"Table processed: {schema.database}.{schema.name}.{view.name}" ) else: - logger.info( - f"No views in cache for {schema.database}.{schema.name}, skipping" + self.report.info( + title="No views found in some schemas", + message="No views found in some schemas. This may be due to insufficient privileges for the provided user.", + context=f"Schema: {schema.database}.{schema.name}", ) else: logger.info("View processing disabled, skipping") @@ -800,11 +823,7 @@ def gen_dataset_workunits( lastModified=( TimeStamp(time=int(table.last_altered.timestamp() * 1000)) if table.last_altered - else ( - TimeStamp(time=int(table.created.timestamp() * 1000)) - if table.created - else None - ) + else None ), description=table.comment, qualifiedName=str(datahub_dataset_name), @@ -1092,3 +1111,43 @@ def add_config_to_report(self): self.config.start_time, self.config.end_time, ) + + def _try_get_redshift_connection( + self, + config: RedshiftConfig, + ) -> Optional[redshift_connector.Connection]: + try: + return RedshiftSource.get_redshift_connection(config) + except redshift_connector.Error as e: + error_message = str(e).lower() + if "password authentication failed" in error_message: + self.report.report_failure( + title="Invalid credentials", + message="Failed to connect to Redshift. Please verify your username, password, and database.", + exc=e, + ) + elif "timeout" in error_message: + self.report.report_failure( + title="Unable to connect", + message="Failed to connect to Redshift. Please verify your host name and port number.", + exc=e, + ) + elif "communication error" in error_message: + self.report.report_failure( + title="Unable to connect", + message="Failed to connect to Redshift. Please verify that the host name is valid and reachable.", + exc=e, + ) + elif "database" in error_message and "does not exist" in error_message: + self.report.report_failure( + title="Database does not exist", + message="Failed to connect to Redshift. Please verify that the provided database exists and the provided user has access to it.", + exc=e, + ) + else: + self.report.report_failure( + title="Unable to connect", + message="Failed to connect to Redshift. Please verify your connection details.", + exc=e, + ) + return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift_schema.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift_schema.py index 101146563e8e7..6e88a50f898a5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift_schema.py @@ -169,6 +169,8 @@ def enrich_tables( self, conn: redshift_connector.Connection, ) -> Dict[str, Dict[str, RedshiftExtraTableMeta]]: + # Warning: This table enrichment will not return anything for + # external tables (spectrum) and for tables that have never been queried / written to. cur = RedshiftDataDictionary.get_query_result( conn, self.queries.additional_table_metadata_query() ) @@ -207,7 +209,7 @@ def get_tables_and_views( # This query needs to run separately as we can't join with the main query because it works with # driver only functions. - enriched_table = self.enrich_tables(conn) + enriched_tables = self.enrich_tables(conn) cur = RedshiftDataDictionary.get_query_result( conn, @@ -216,6 +218,7 @@ def get_tables_and_views( field_names = [i[0] for i in cur.description] db_tables = cur.fetchall() logger.info(f"Fetched {len(db_tables)} tables/views from Redshift") + for table in db_tables: schema = table[field_names.index("schema")] table_name = table[field_names.index("relname")] @@ -233,7 +236,7 @@ def get_tables_and_views( rows_count, size_in_bytes, ) = RedshiftDataDictionary.get_table_stats( - enriched_table, field_names, schema, table + enriched_tables, field_names, schema, table ) tables[schema].append( @@ -263,15 +266,15 @@ def get_tables_and_views( rows_count, size_in_bytes, ) = RedshiftDataDictionary.get_table_stats( - enriched_table=enriched_table, + enriched_tables=enriched_tables, field_names=field_names, schema=schema, table=table, ) materialized = False - if schema in enriched_table and table_name in enriched_table[schema]: - if enriched_table[schema][table_name].is_materialized: + if schema in enriched_tables and table_name in enriched_tables[schema]: + if enriched_tables[schema][table_name].is_materialized: materialized = True views[schema].append( @@ -298,7 +301,7 @@ def get_tables_and_views( return tables, views @staticmethod - def get_table_stats(enriched_table, field_names, schema, table): + def get_table_stats(enriched_tables, field_names, schema, table): table_name = table[field_names.index("relname")] creation_time: Optional[datetime] = None @@ -309,25 +312,41 @@ def get_table_stats(enriched_table, field_names, schema, table): last_altered: Optional[datetime] = None size_in_bytes: Optional[int] = None rows_count: Optional[int] = None - if schema in enriched_table and table_name in enriched_table[schema]: - if enriched_table[schema][table_name].last_accessed: + if schema in enriched_tables and table_name in enriched_tables[schema]: + if enriched_tables[schema][table_name].last_accessed is not None: # Mypy seems to be not clever enough to understand the above check - last_accessed = enriched_table[schema][table_name].last_accessed + last_accessed = enriched_tables[schema][table_name].last_accessed assert last_accessed last_altered = last_accessed.replace(tzinfo=timezone.utc) elif creation_time: last_altered = creation_time - if enriched_table[schema][table_name].size: + if enriched_tables[schema][table_name].size is not None: # Mypy seems to be not clever enough to understand the above check - size = enriched_table[schema][table_name].size + size = enriched_tables[schema][table_name].size if size: size_in_bytes = size * 1024 * 1024 - if enriched_table[schema][table_name].estimated_visible_rows: - rows = enriched_table[schema][table_name].estimated_visible_rows + if enriched_tables[schema][table_name].estimated_visible_rows is not None: + rows = enriched_tables[schema][table_name].estimated_visible_rows assert rows rows_count = int(rows) + else: + # The object was not found in the enriched data. + # + # If we don't have enriched data, it may be either because: + # 1 The table is empty (as per https://docs.aws.amazon.com/redshift/latest/dg/r_SVV_TABLE_INFO.html) empty tables are omitted from svv_table_info. + # 2. The table is external + # 3. The table is a view (non-materialized) + # + # In case 1, we want to report an accurate profile suggesting that the table is empty. + # In case 2, do nothing since we cannot cheaply profile + # In case 3, do nothing since we cannot cheaply profile + if table[field_names.index("tabletype")] == "TABLE": + rows_count = 0 + size_in_bytes = 0 + logger.info("Found some tables with no profiles need to return 0") + return creation_time, last_altered, rows_count, size_in_bytes @staticmethod diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/report.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/report.py index 2e6cb8051c91e..ff28ed2c5e849 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/report.py @@ -44,6 +44,8 @@ class RedshiftReport( num_lineage_dropped_query_parser: int = 0 num_lineage_dropped_not_support_copy_path: int = 0 num_lineage_processed_temp_tables = 0 + num_lineage_dropped_s3_path: int = 0 + num_alter_table_parse_errors: int = 0 lineage_start_time: Optional[datetime] = None lineage_end_time: Optional[datetime] = None diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py index 8bc075f720cc5..c35f500df1b8c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py @@ -217,7 +217,7 @@ class TableData: number_of_files: int -@platform_name("S3 Data Lake", id="s3") +@platform_name("S3 / Local Files", id="s3") @config_class(DataLakeSourceConfig) @support_status(SupportStatus.INCUBATING) @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration") diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/config.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/config.py index 06c381add05b9..6a47884e1b139 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sigma/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/config.py @@ -1,5 +1,5 @@ import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Dict, Optional import pydantic @@ -26,7 +26,10 @@ class Constant: """ # Rest API response key constants + REFRESH_TOKEN = "refresh_token" + ACCESS_TOKEN = "access_token" ENTRIES = "entries" + MEMBERID = "memberId" FIRSTNAME = "firstName" LASTNAME = "lastName" EDGES = "edges" @@ -52,6 +55,11 @@ class Constant: @dataclass class SigmaSourceReport(StaleEntityRemovalSourceReport): number_of_workspaces: int = 0 + non_accessible_workspaces_count: int = 0 + shared_entities_count: int = 0 + number_of_datasets: int = 0 + number_of_workbooks: int = 0 + number_of_files_metadata: Dict[str, int] = field(default_factory=dict) def report_number_of_workspaces(self, number_of_workspaces: int) -> None: self.number_of_workspaces = number_of_workspaces @@ -75,15 +83,29 @@ class SigmaSourceConfig( workspace_pattern: AllowDenyPattern = pydantic.Field( default=AllowDenyPattern.allow_all(), description="Regex patterns to filter Sigma workspaces in ingestion." - "Mention 'User Folder' if entities of 'My documents' need to ingest.", + "Mention 'My documents' if personal entities also need to ingest.", ) ingest_owner: Optional[bool] = pydantic.Field( default=True, - description="Ingest Owner from source. This will override Owner info entered from UI", + description="Ingest Owner from source. This will override Owner info entered from UI.", + ) + ingest_shared_entities: Optional[bool] = pydantic.Field( + default=False, + description="Whether to ingest the shared entities or not.", + ) + extract_lineage: Optional[bool] = pydantic.Field( + default=True, + description="Whether to extract lineage of workbook's elements and datasets or not.", + ) + workbook_lineage_pattern: AllowDenyPattern = pydantic.Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns to filter workbook's elements and datasets lineage in ingestion." + "Requires extract_lineage to be enabled.", ) chart_sources_platform_mapping: Dict[str, PlatformDetail] = pydantic.Field( default={}, description="A mapping of the sigma workspace/workbook/chart folder path to all chart's data sources platform details present inside that folder path.", ) - - stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None + stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = pydantic.Field( + default=None, description="Sigma Stateful Ingestion Config." + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py index 9863adc4a854a..922b0be3b4a93 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py @@ -21,46 +21,56 @@ class Workspace(BaseModel): createdAt: datetime updatedAt: datetime + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + # Update name if presonal workspace + if values["name"] == "User Folder": + values["name"] = "My documents" + return values + class SigmaDataset(BaseModel): datasetId: str - workspaceId: str name: str description: str createdBy: str createdAt: datetime updatedAt: datetime url: str - path: str + workspaceId: Optional[str] = None + path: Optional[str] = None badge: Optional[str] = None - @root_validator(pre=True) - def update_values(cls, values: Dict) -> Dict: + def get_urn_part(self): # As element lineage api provide this id as source dataset id - values["datasetId"] = values["url"].split("/")[-1] - return values + return self.url.split("/")[-1] class Element(BaseModel): elementId: str - type: str name: str url: str + type: Optional[str] = None vizualizationType: Optional[str] = None query: Optional[str] = None columns: List[str] = [] upstream_sources: Dict[str, str] = {} + def get_urn_part(self): + return self.elementId + class Page(BaseModel): pageId: str name: str elements: List[Element] = [] + def get_urn_part(self): + return self.pageId + class Workbook(BaseModel): workbookId: str - workspaceId: str name: str createdBy: str updatedBy: str @@ -69,5 +79,16 @@ class Workbook(BaseModel): url: str path: str latestVersion: int + workspaceId: Optional[str] = None pages: List[Page] = [] badge: Optional[str] = None + + +class File(BaseModel): + id: str + name: str + parentId: str + path: str + type: str + badge: Optional[str] = None + workspaceId: Optional[str] = None diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py index 97da198aaa435..74d7abb121a3e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py @@ -110,7 +110,7 @@ def __init__(self, config: SigmaSourceConfig, ctx: PipelineContext): self.reporter = SigmaSourceReport() self.dataset_upstream_urn_mapping: Dict[str, List[str]] = {} try: - self.sigma_api = SigmaAPI(self.config) + self.sigma_api = SigmaAPI(self.config, self.reporter) except Exception as e: raise ConfigurationError(f"Unable to connect sigma API. Exception: {e}") @@ -118,7 +118,10 @@ def __init__(self, config: SigmaSourceConfig, ctx: PipelineContext): def test_connection(config_dict: dict) -> TestConnectionReport: test_report = TestConnectionReport() try: - SigmaAPI(SigmaSourceConfig.parse_obj_allow_extras(config_dict)) + SigmaAPI( + SigmaSourceConfig.parse_obj_allow_extras(config_dict), + SigmaSourceReport(), + ) test_report.basic_connectivity = CapabilityReport(capable=True) except Exception as e: test_report.basic_connectivity = CapabilityReport( @@ -175,9 +178,6 @@ def _gen_workspace_workunit( last_modified=int(workspace.updatedAt.timestamp() * 1000), ) - def _get_sigma_dataset_identifier(self, dataset: SigmaDataset) -> str: - return dataset.datasetId - def _gen_sigma_dataset_urn(self, dataset_identifier: str) -> str: return builder.make_dataset_urn_with_platform_instance( name=dataset_identifier, @@ -201,9 +201,11 @@ def _gen_dataset_properties( externalUrl=dataset.url, created=TimeStamp(time=int(dataset.createdAt.timestamp() * 1000)), lastModified=TimeStamp(time=int(dataset.updatedAt.timestamp() * 1000)), + customProperties={"datasetId": dataset.datasetId}, tags=[dataset.badge] if dataset.badge else None, ) - dataset_properties.customProperties.update({"path": dataset.path}) + if dataset.path: + dataset_properties.customProperties["path"] = dataset.path return MetadataChangeProposalWrapper( entityUrn=dataset_urn, aspect=dataset_properties ).as_workunit() @@ -262,18 +264,18 @@ def _gen_entity_browsepath_aspect( def _gen_dataset_workunit( self, dataset: SigmaDataset ) -> Iterable[MetadataWorkUnit]: - dataset_identifier = self._get_sigma_dataset_identifier(dataset) - dataset_urn = self._gen_sigma_dataset_urn(dataset_identifier) + dataset_urn = self._gen_sigma_dataset_urn(dataset.get_urn_part()) yield self._gen_entity_status_aspect(dataset_urn) yield self._gen_dataset_properties(dataset_urn, dataset) - yield from add_entity_to_container( - container_key=self._gen_workspace_key(dataset.workspaceId), - entity_type="dataset", - entity_urn=dataset_urn, - ) + if dataset.workspaceId: + yield from add_entity_to_container( + container_key=self._gen_workspace_key(dataset.workspaceId), + entity_type="dataset", + entity_urn=dataset_urn, + ) dpi_aspect = self._gen_dataplatform_instance_aspect(dataset_urn) if dpi_aspect: @@ -288,15 +290,16 @@ def _gen_dataset_workunit( aspect=SubTypes(typeNames=[DatasetSubTypes.SIGMA_DATASET]), ).as_workunit() - paths = dataset.path.split("/")[1:] - if len(paths) > 0: - yield self._gen_entity_browsepath_aspect( - entity_urn=dataset_urn, - parent_entity_urn=builder.make_container_urn( - self._gen_workspace_key(dataset.workspaceId) - ), - paths=paths, - ) + if dataset.path and dataset.workspaceId: + paths = dataset.path.split("/")[1:] + if len(paths) > 0: + yield self._gen_entity_browsepath_aspect( + entity_urn=dataset_urn, + parent_entity_urn=builder.make_container_urn( + self._gen_workspace_key(dataset.workspaceId) + ), + paths=paths, + ) if dataset.badge: yield MetadataChangeProposalWrapper( @@ -322,7 +325,7 @@ def _gen_dashboard_urn(self, dashboard_identifier: str) -> str: ) def _gen_dashboard_info_workunit(self, page: Page) -> MetadataWorkUnit: - dashboard_urn = self._gen_dashboard_urn(page.pageId) + dashboard_urn = self._gen_dashboard_urn(page.get_urn_part()) dashboard_info_cls = DashboardInfoClass( title=page.name, description="", @@ -330,7 +333,7 @@ def _gen_dashboard_info_workunit(self, page: Page) -> MetadataWorkUnit: builder.make_chart_urn( platform=self.platform, platform_instance=self.config.platform_instance, - name=element.elementId, + name=element.get_urn_part(), ) for element in page.elements ], @@ -424,12 +427,12 @@ def _gen_elements_workunit( chart_urn = builder.make_chart_urn( platform=self.platform, platform_instance=self.config.platform_instance, - name=element.elementId, + name=element.get_urn_part(), ) custom_properties = { "VizualizationType": str(element.vizualizationType), - "type": str(element.type), + "type": str(element.type) if element.type else "Unknown", } yield self._gen_entity_status_aspect(chart_urn) @@ -490,7 +493,7 @@ def _gen_pages_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUnit]: Map Sigma workbook page to Datahub dashboard """ for page in workbook.pages: - dashboard_urn = self._gen_dashboard_urn(page.pageId) + dashboard_urn = self._gen_dashboard_urn(page.get_urn_part()) yield self._gen_entity_status_aspect(dashboard_urn) @@ -513,7 +516,7 @@ def _gen_pages_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUnit]: ) yield MetadataChangeProposalWrapper( - entityUrn=self._gen_dashboard_urn(page.pageId), + entityUrn=dashboard_urn, aspect=InputFieldsClass(fields=all_input_fields), ).as_workunit() @@ -522,11 +525,14 @@ def _gen_workbook_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUni Map Sigma Workbook to Datahub container """ owner_username = self.sigma_api.get_user_name(workbook.createdBy) + workbook_key = self._gen_workbook_key(workbook.workbookId) yield from gen_containers( - container_key=self._gen_workbook_key(workbook.workbookId), + container_key=workbook_key, name=workbook.name, sub_types=[BIContainerSubTypes.SIGMA_WORKBOOK], - parent_container_key=self._gen_workspace_key(workbook.workspaceId), + parent_container_key=self._gen_workspace_key(workbook.workspaceId) + if workbook.workspaceId + else None, extra_properties={ "path": workbook.path, "latestVersion": str(workbook.latestVersion), @@ -541,11 +547,9 @@ def _gen_workbook_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUni ) paths = workbook.path.split("/")[1:] - if len(paths) > 0: + if len(paths) > 0 and workbook.workspaceId: yield self._gen_entity_browsepath_aspect( - entity_urn=builder.make_container_urn( - self._gen_workbook_key(workbook.workbookId), - ), + entity_urn=builder.make_container_urn(workbook_key), parent_entity_urn=builder.make_container_urn( self._gen_workspace_key(workbook.workspaceId) ), @@ -578,12 +582,13 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: Datahub Ingestion framework invoke this method """ logger.info("Sigma plugin execution is started") - entities = self.sigma_api.get_sigma_entities() - for entity in entities: - if isinstance(entity, Workbook): - yield from self._gen_workbook_workunit(entity) - elif isinstance(entity, SigmaDataset): - yield from self._gen_dataset_workunit(entity) + self.sigma_api.fill_workspaces() + + for dataset in self.sigma_api.get_sigma_datasets(): + yield from self._gen_dataset_workunit(dataset) + for workbook in self.sigma_api.get_sigma_workbooks(): + yield from self._gen_workbook_workunit(workbook) + for workspace in self._get_allowed_workspaces(): yield from self._gen_workspace_workunit(workspace) yield from self._gen_sigma_dataset_upstream_lineage_workunit() diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma_api.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma_api.py index c2c28419ebcfd..66d4678e52132 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma_api.py @@ -1,12 +1,18 @@ +import functools import logging import sys -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional import requests -from datahub.ingestion.source.sigma.config import Constant, SigmaSourceConfig +from datahub.ingestion.source.sigma.config import ( + Constant, + SigmaSourceConfig, + SigmaSourceReport, +) from datahub.ingestion.source.sigma.data_classes import ( Element, + File, Page, SigmaDataset, Workbook, @@ -18,11 +24,13 @@ class SigmaAPI: - def __init__(self, config: SigmaSourceConfig) -> None: + def __init__(self, config: SigmaSourceConfig, report: SigmaSourceReport) -> None: self.config = config + self.report = report self.workspaces: Dict[str, Workspace] = {} self.users: Dict[str, str] = {} self.session = requests.Session() + self.refresh_token: Optional[str] = None # Test connection by generating access token logger.info(f"Trying to connect to {self.config.api_url}") self._generate_token() @@ -35,84 +43,235 @@ def _generate_token(self): } response = self.session.post(f"{self.config.api_url}/auth/token", data=data) response.raise_for_status() + response_dict = response.json() + self.refresh_token = response_dict[Constant.REFRESH_TOKEN] self.session.headers.update( { - "Authorization": f"Bearer {response.json()['access_token']}", + "Authorization": f"Bearer {response_dict[Constant.ACCESS_TOKEN]}", "Content-Type": "application/json", } ) def _log_http_error(self, message: str) -> Any: - logger.warning(message) _, e, _ = sys.exc_info() if isinstance(e, requests.exceptions.HTTPError): logger.warning(f"HTTP status-code = {e.response.status_code}") logger.debug(msg=message, exc_info=e) return e - def get_workspace(self, workspace_id: str) -> Optional[Workspace]: - workspace: Optional[Workspace] = None + def _refresh_access_token(self): try: - response = self.session.get( - f"{self.config.api_url}/workspaces/{workspace_id}" + data = { + "grant_type": Constant.REFRESH_TOKEN, + "refresh_token": self.refresh_token, + "client_id": self.config.client_id, + "client_secret": self.config.client_secret, + } + post_response = self.session.post( + f"{self.config.api_url}/auth/token", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + data=data, + ) + post_response.raise_for_status() + response_dict = post_response.json() + self.refresh_token = response_dict[Constant.REFRESH_TOKEN] + self.session.headers.update( + { + "Authorization": f"Bearer {response_dict[Constant.ACCESS_TOKEN]}", + "Content-Type": "application/json", + } ) - response.raise_for_status() - workspace_dict = response.json() - workspace = Workspace.parse_obj(workspace_dict) except Exception as e: self._log_http_error( - message=f"Unable to fetch workspace {workspace_id}. Exception: {e}" + message=f"Unable to refresh access token. Exception: {e}" ) - return workspace - def get_user_name(self, user_id: str) -> Optional[str]: + def _get_api_call(self, url: str) -> requests.Response: + get_response = self.session.get(url) + if get_response.status_code == 401 and self.refresh_token: + logger.debug("Access token might expired. Refreshing access token.") + self._refresh_access_token() + get_response = self.session.get(url) + return get_response + + def get_workspace(self, workspace_id: str) -> Optional[Workspace]: + logger.debug(f"Fetching workspace metadata with id '{workspace_id}'") try: - if user_id in self.users: - # To avoid fetching same user details again - return self.users[user_id] + if workspace_id in self.workspaces: + return self.workspaces[workspace_id] else: - response = self.session.get(f"{self.config.api_url}/members/{user_id}") - response.raise_for_status() - user_dict = response.json() - user_name = ( - f"{user_dict[Constant.FIRSTNAME]}_{user_dict[Constant.LASTNAME]}" + response = self._get_api_call( + f"{self.config.api_url}/workspaces/{workspace_id}" ) - self.users[user_id] = user_name - return user_name + if response.status_code == 403: + logger.debug(f"Workspace {workspace_id} not accessible.") + self.report.non_accessible_workspaces_count += 1 + return None + response.raise_for_status() + workspace = Workspace.parse_obj(response.json()) + self.workspaces[workspace.workspaceId] = workspace + return workspace except Exception as e: self._log_http_error( - message=f"Unable to fetch user with id {user_id}. Exception: {e}" + message=f"Unable to fetch workspace '{workspace_id}'. Exception: {e}" ) return None - def get_sigma_dataset( - self, dataset_id: str, workspace_id: str, path: str - ) -> Optional[SigmaDataset]: - dataset: Optional[SigmaDataset] = None + def fill_workspaces(self) -> None: + logger.debug("Fetching all accessible workspaces metadata.") + workspace_url = url = f"{self.config.api_url}/workspaces?limit=50" try: - response = self.session.get(f"{self.config.api_url}/datasets/{dataset_id}") + while True: + response = self._get_api_call(url) + response.raise_for_status() + response_dict = response.json() + for workspace_dict in response_dict[Constant.ENTRIES]: + self.workspaces[ + workspace_dict[Constant.WORKSPACEID] + ] = Workspace.parse_obj(workspace_dict) + if response_dict[Constant.NEXTPAGE]: + url = f"{workspace_url}&page={response_dict[Constant.NEXTPAGE]}" + else: + break + except Exception as e: + self._log_http_error(message=f"Unable to fetch workspaces. Exception: {e}") + + @functools.lru_cache() + def _get_users(self) -> Dict[str, str]: + logger.debug("Fetching all accessible users metadata.") + try: + users: Dict[str, str] = {} + response = self._get_api_call(f"{self.config.api_url}/members") response.raise_for_status() - dataset_dict = response.json() - dataset_dict[Constant.WORKSPACEID] = workspace_id - dataset_dict[Constant.PATH] = path - dataset = SigmaDataset.parse_obj(dataset_dict) + for user_dict in response.json(): + users[ + user_dict[Constant.MEMBERID] + ] = f"{user_dict[Constant.FIRSTNAME]}_{user_dict[Constant.LASTNAME]}" + return users + except Exception as e: + self._log_http_error( + message=f"Unable to fetch users details. Exception: {e}" + ) + return {} + + def get_user_name(self, user_id: str) -> Optional[str]: + return self._get_users().get(user_id) + + @functools.lru_cache() + def get_workspace_id_from_file_path( + self, parent_id: str, path: str + ) -> Optional[str]: + try: + path_list = path.split("/") + while len(path_list) != 1: # means current parent id is folder's id + response = self._get_api_call( + f"{self.config.api_url}/files/{parent_id}" + ) + response.raise_for_status() + parent_id = response.json()[Constant.PARENTID] + path_list.pop() + return parent_id + except Exception as e: + logger.error( + f"Unable to find workspace id using file path '{path}'. Exception: {e}" + ) + return None + + @functools.lru_cache + def _get_files_metadata(self, file_type: str) -> Dict[str, File]: + logger.debug(f"Fetching file metadata with type {file_type}.") + file_url = url = f"{self.config.api_url}/files?typeFilters={file_type}" + try: + files_metadata: Dict[str, File] = {} + while True: + response = self._get_api_call(url) + response.raise_for_status() + response_dict = response.json() + for file_dict in response_dict[Constant.ENTRIES]: + file = File.parse_obj(file_dict) + file.workspaceId = self.get_workspace_id_from_file_path( + file.parentId, file.path + ) + files_metadata[file_dict[Constant.ID]] = file + if response_dict[Constant.NEXTPAGE]: + url = f"{file_url}&page={response_dict[Constant.NEXTPAGE]}" + else: + break + self.report.number_of_files_metadata[file_type] = len(files_metadata) + return files_metadata + except Exception as e: + self._log_http_error( + message=f"Unable to fetch files metadata. Exception: {e}" + ) + return {} + + def get_sigma_datasets(self) -> List[SigmaDataset]: + logger.debug("Fetching all accessible datasets metadata.") + dataset_url = url = f"{self.config.api_url}/datasets" + dataset_files_metadata = self._get_files_metadata(file_type=Constant.DATASET) + try: + datasets: List[SigmaDataset] = [] + while True: + response = self._get_api_call(url) + response.raise_for_status() + response_dict = response.json() + for dataset_dict in response_dict[Constant.ENTRIES]: + dataset = SigmaDataset.parse_obj(dataset_dict) + + if dataset.datasetId in dataset_files_metadata: + dataset.path = dataset_files_metadata[dataset.datasetId].path + dataset.badge = dataset_files_metadata[dataset.datasetId].badge + + workspace_id = dataset_files_metadata[ + dataset.datasetId + ].workspaceId + if workspace_id: + dataset.workspaceId = workspace_id + workspace = self.get_workspace(dataset.workspaceId) + if workspace: + if self.config.workspace_pattern.allowed( + workspace.name + ): + datasets.append(dataset) + elif self.config.ingest_shared_entities: + # If no workspace for dataset we can consider it as shared entity + self.report.shared_entities_count += 1 + datasets.append(dataset) + + if response_dict[Constant.NEXTPAGE]: + url = f"{dataset_url}?page={response_dict[Constant.NEXTPAGE]}" + else: + break + self.report.number_of_datasets = len(datasets) + return datasets except Exception as e: self._log_http_error( - message=f"Unable to fetch sigma dataset {dataset_id}. Exception: {e}" + message=f"Unable to fetch sigma datasets. Exception: {e}" ) - return dataset + return [] def _get_element_upstream_sources( - self, element_id: str, workbook_id: str + self, element: Element, workbook: Workbook ) -> Dict[str, str]: """ Returns upstream dataset sources with keys as id and values as name of that dataset """ - upstream_sources: Dict[str, str] = {} try: - response = self.session.get( - f"{self.config.api_url}/workbooks/{workbook_id}/lineage/elements/{element_id}" + upstream_sources: Dict[str, str] = {} + response = self._get_api_call( + f"{self.config.api_url}/workbooks/{workbook.workbookId}/lineage/elements/{element.elementId}" ) + if response.status_code == 500: + logger.debug( + f"Lineage metadata not present for element {element.name} of workbook '{workbook.name}'" + ) + return upstream_sources + if response.status_code == 403: + logger.debug( + f"Lineage metadata not accessible for element {element.name} of workbook '{workbook.name}'" + ) + return upstream_sources + response.raise_for_status() response_dict = response.json() for edge in response_dict[Constant.EDGES]: @@ -123,34 +282,39 @@ def _get_element_upstream_sources( upstream_sources[edge[Constant.SOURCE]] = response_dict[ Constant.DEPENDENCIES ][edge[Constant.SOURCE]][Constant.NAME] + return upstream_sources except Exception as e: self._log_http_error( - message=f"Unable to fetch lineage of element {element_id}. Exception: {e}" + message=f"Unable to fetch lineage for element {element.name} of workbook '{workbook.name}'. Exception: {e}" ) - return upstream_sources + return {} def _get_element_sql_query( - self, element_id: str, workbook_id: str + self, element: Element, workbook: Workbook ) -> Optional[str]: - query: Optional[str] = None try: - response = self.session.get( - f"{self.config.api_url}/workbooks/{workbook_id}/elements/{element_id}/query" + response = self._get_api_call( + f"{self.config.api_url}/workbooks/{workbook.workbookId}/elements/{element.elementId}/query" ) + if response.status_code == 404: + logger.debug( + f"Query not present for element {element.name} of workbook '{workbook.name}'" + ) + return None response.raise_for_status() response_dict = response.json() if "sql" in response_dict: - query = response_dict["sql"] + return response_dict["sql"] except Exception as e: self._log_http_error( - message=f"Unable to fetch sql query for a element {element_id}. Exception: {e}" + message=f"Unable to fetch sql query for element {element.name} of workbook '{workbook.name}'. Exception: {e}" ) - return query + return None def get_page_elements(self, workbook: Workbook, page: Page) -> List[Element]: - elements: List[Element] = [] try: - response = self.session.get( + elements: List[Element] = [] + response = self._get_api_call( f"{self.config.api_url}/workbooks/{workbook.workbookId}/pages/{page.pageId}/elements" ) response.raise_for_status() @@ -161,23 +325,26 @@ def get_page_elements(self, workbook: Workbook, page: Page) -> List[Element]: Constant.URL ] = f"{workbook.url}?:nodeId={element_dict[Constant.ELEMENTID]}&:fullScreen=true" element = Element.parse_obj(element_dict) - element.upstream_sources = self._get_element_upstream_sources( - element.elementId, workbook.workbookId - ) - element.query = self._get_element_sql_query( - element.elementId, workbook.workbookId - ) + if ( + self.config.extract_lineage + and self.config.workbook_lineage_pattern.allowed(workbook.name) + ): + element.upstream_sources = self._get_element_upstream_sources( + element, workbook + ) + element.query = self._get_element_sql_query(element, workbook) elements.append(element) + return elements except Exception as e: self._log_http_error( - message=f"Unable to fetch elements of page {page.pageId}, workbook {workbook.workbookId}. Exception: {e}" + message=f"Unable to fetch elements of page '{page.name}', workbook '{workbook.name}'. Exception: {e}" ) - return elements + return [] def get_workbook_pages(self, workbook: Workbook) -> List[Page]: - pages: List[Page] = [] try: - response = self.session.get( + pages: List[Page] = [] + response = self._get_api_call( f"{self.config.api_url}/workbooks/{workbook.workbookId}/pages" ) response.raise_for_status() @@ -185,75 +352,57 @@ def get_workbook_pages(self, workbook: Workbook) -> List[Page]: page = Page.parse_obj(page_dict) page.elements = self.get_page_elements(workbook, page) pages.append(page) + return pages except Exception as e: self._log_http_error( - message=f"Unable to fetch pages of workbook {workbook.workbookId}. Exception: {e}" + message=f"Unable to fetch pages of workbook '{workbook.name}'. Exception: {e}" ) - return pages + return [] - def get_workbook(self, workbook_id: str, workspace_id: str) -> Optional[Workbook]: - workbook: Optional[Workbook] = None + def get_sigma_workbooks(self) -> List[Workbook]: + logger.debug("Fetching all accessible workbooks metadata.") + workbook_url = url = f"{self.config.api_url}/workbooks" + workbook_files_metadata = self._get_files_metadata(file_type=Constant.WORKBOOK) try: - response = self.session.get( - f"{self.config.api_url}/workbooks/{workbook_id}" - ) - response.raise_for_status() - workbook_dict = response.json() - workbook_dict[Constant.WORKSPACEID] = workspace_id - workbook = Workbook.parse_obj(workbook_dict) - workbook.pages = self.get_workbook_pages(workbook) - except Exception as e: - self._log_http_error( - message=f"Unable to fetch workbook {workbook_id}. Exception: {e}" - ) - return workbook + workbooks: List[Workbook] = [] + while True: + response = self._get_api_call(url) + response.raise_for_status() + response_dict = response.json() + for workbook_dict in response_dict[Constant.ENTRIES]: + workbook = Workbook.parse_obj(workbook_dict) - def get_workspace_id(self, parent_id: str, path: str) -> str: - path_list = path.split("/") - while len(path_list) != 1: # means current parent id is folder's id - response = self.session.get(f"{self.config.api_url}/files/{parent_id}") - parent_id = response.json()[Constant.PARENTID] - path_list.pop() - return parent_id + if workbook.workbookId in workbook_files_metadata: + workbook.badge = workbook_files_metadata[ + workbook.workbookId + ].badge - def get_sigma_entities(self) -> List[Union[Workbook, SigmaDataset]]: - entities: List[Union[Workbook, SigmaDataset]] = [] - url = f"{self.config.api_url}/files" - while True: - response = self.session.get(url) - response.raise_for_status() - response_dict = response.json() - for entity in response_dict[Constant.ENTRIES]: - workspace_id = self.get_workspace_id( - entity[Constant.PARENTID], entity[Constant.PATH] - ) - if workspace_id not in self.workspaces: - workspace = self.get_workspace(workspace_id) - if workspace: - self.workspaces[workspace.workspaceId] = workspace + workspace_id = workbook_files_metadata[ + workbook.workbookId + ].workspaceId + if workspace_id: + workbook.workspaceId = workspace_id + workspace = self.get_workspace(workbook.workspaceId) + if workspace: + if self.config.workspace_pattern.allowed( + workspace.name + ): + workbook.pages = self.get_workbook_pages(workbook) + workbooks.append(workbook) + elif self.config.ingest_shared_entities: + # If no workspace for workbook we can consider it as shared entity + self.report.shared_entities_count += 1 + workbook.pages = self.get_workbook_pages(workbook) + workbooks.append(workbook) - if self.workspaces.get( - workspace_id - ) and self.config.workspace_pattern.allowed( - self.workspaces[workspace_id].name - ): - type = entity[Constant.TYPE] - if type == Constant.DATASET: - dataset = self.get_sigma_dataset( - entity[Constant.ID], - workspace_id, - entity[Constant.PATH], - ) - if dataset: - dataset.badge = entity[Constant.BADGE] - entities.append(dataset) - elif type == Constant.WORKBOOK: - workbook = self.get_workbook(entity[Constant.ID], workspace_id) - if workbook: - workbook.badge = entity[Constant.BADGE] - entities.append(workbook) - if response_dict[Constant.NEXTPAGE]: - url = f"{url}?page={response_dict[Constant.NEXTPAGE]}" - else: - break - return entities + if response_dict[Constant.NEXTPAGE]: + url = f"{workbook_url}?page={response_dict[Constant.NEXTPAGE]}" + else: + break + self.report.number_of_workbooks = len(workbooks) + return workbooks + except Exception as e: + self._log_http_error( + message=f"Unable to fetch sigma workbooks. Exception: {e}" + ) + return [] diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_assertion.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_assertion.py new file mode 100644 index 0000000000000..2a1d18c83e6fa --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_assertion.py @@ -0,0 +1,124 @@ +import logging +from datetime import datetime +from typing import Iterable, List, Optional + +from pydantic import BaseModel + +from datahub.emitter.mce_builder import ( + make_assertion_urn, + make_data_platform_urn, + make_dataplatform_instance_urn, +) +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.snowflake.snowflake_config import ( + SnowflakeIdentifierConfig, + SnowflakeV2Config, +) +from datahub.ingestion.source.snowflake.snowflake_connection import SnowflakeConnection +from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery +from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report +from datahub.ingestion.source.snowflake.snowflake_utils import SnowflakeIdentifierMixin +from datahub.metadata.com.linkedin.pegasus2avro.assertion import ( + AssertionResult, + AssertionResultType, + AssertionRunEvent, + AssertionRunStatus, +) +from datahub.metadata.com.linkedin.pegasus2avro.common import DataPlatformInstance +from datahub.utilities.time import datetime_to_ts_millis + +logger: logging.Logger = logging.getLogger(__name__) + + +class DataQualityMonitoringResult(BaseModel): + MEASUREMENT_TIME: datetime + METRIC_NAME: str + TABLE_NAME: str + TABLE_SCHEMA: str + TABLE_DATABASE: str + VALUE: int + + +class SnowflakeAssertionsHandler(SnowflakeIdentifierMixin): + def __init__( + self, + config: SnowflakeV2Config, + report: SnowflakeV2Report, + connection: SnowflakeConnection, + ) -> None: + self.config = config + self.report = report + self.logger = logger + self.connection = connection + self._urns_processed: List[str] = [] + + @property + def identifier_config(self) -> SnowflakeIdentifierConfig: + return self.config + + def get_assertion_workunits( + self, discovered_datasets: List[str] + ) -> Iterable[MetadataWorkUnit]: + cur = self.connection.query( + SnowflakeQuery.dmf_assertion_results( + datetime_to_ts_millis(self.config.start_time), + datetime_to_ts_millis(self.config.end_time), + ) + ) + for db_row in cur: + mcp = self._process_result_row(db_row, discovered_datasets) + if mcp: + yield mcp.as_workunit(is_primary_source=False) + + if mcp.entityUrn and mcp.entityUrn not in self._urns_processed: + self._urns_processed.append(mcp.entityUrn) + yield self._gen_platform_instance_wu(mcp.entityUrn) + + def _gen_platform_instance_wu(self, urn: str) -> MetadataWorkUnit: + # Construct a MetadataChangeProposalWrapper object for assertion platform + return MetadataChangeProposalWrapper( + entityUrn=urn, + aspect=DataPlatformInstance( + platform=make_data_platform_urn(self.platform), + instance=( + make_dataplatform_instance_urn( + self.platform, self.config.platform_instance + ) + if self.config.platform_instance + else None + ), + ), + ).as_workunit(is_primary_source=False) + + def _process_result_row( + self, result_row: dict, discovered_datasets: List[str] + ) -> Optional[MetadataChangeProposalWrapper]: + try: + result = DataQualityMonitoringResult.parse_obj(result_row) + assertion_guid = result.METRIC_NAME.split("__")[-1].lower() + status = bool(result.VALUE) # 1 if PASS, 0 if FAIL + assertee = self.get_dataset_identifier( + result.TABLE_NAME, result.TABLE_SCHEMA, result.TABLE_DATABASE + ) + if assertee in discovered_datasets: + return MetadataChangeProposalWrapper( + entityUrn=make_assertion_urn(assertion_guid), + aspect=AssertionRunEvent( + timestampMillis=datetime_to_ts_millis(result.MEASUREMENT_TIME), + runId=result.MEASUREMENT_TIME.strftime("%Y-%m-%dT%H:%M:%SZ"), + asserteeUrn=self.gen_dataset_urn(assertee), + status=AssertionRunStatus.COMPLETE, + assertionUrn=make_assertion_urn(assertion_guid), + result=AssertionResult( + type=( + AssertionResultType.SUCCESS + if status + else AssertionResultType.FAILURE + ) + ), + ), + ) + except Exception as e: + self.report.report_warning("assertion-result-parse-failure", str(e)) + return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py index c1fbb2cdc1f3f..f6247eb949417 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py @@ -4,24 +4,31 @@ from enum import Enum from typing import Dict, List, Optional, Set, cast +import pydantic from pydantic import Field, SecretStr, root_validator, validator from datahub.configuration.common import AllowDenyPattern, ConfigModel from datahub.configuration.pattern_utils import UUID_REGEX +from datahub.configuration.source_common import ( + EnvConfigMixin, + LowerCaseDatasetUrnConfigMixin, + PlatformInstanceConfigMixin, +) +from datahub.configuration.time_window_config import BaseTimeWindowConfig from datahub.configuration.validate_field_removal import pydantic_removed_field from datahub.configuration.validate_field_rename import pydantic_renamed_field from datahub.ingestion.glossary.classification_mixin import ( ClassificationSourceConfigMixin, ) +from datahub.ingestion.source.snowflake.snowflake_connection import ( + SnowflakeConnectionConfig, +) +from datahub.ingestion.source.sql.sql_config import SQLCommonConfig, SQLFilterConfig from datahub.ingestion.source.state.stateful_ingestion_base import ( StatefulLineageConfigMixin, StatefulProfilingConfigMixin, StatefulUsageConfigMixin, ) -from datahub.ingestion.source_config.sql.snowflake import ( - BaseSnowflakeConfig, - SnowflakeConfig, -) from datahub.ingestion.source_config.usage.snowflake_usage import SnowflakeUsageConfig from datahub.utilities.global_warning_util import add_global_warning @@ -32,11 +39,12 @@ # # DBT incremental models create temporary tables ending with __dbt_tmp # Ref - https://discourse.getdbt.com/t/handling-bigquery-incremental-dbt-tmp-tables/7540 -DEFAULT_TABLES_DENY_LIST = [ +DEFAULT_TEMP_TABLES_PATTERNS = [ r".*\.FIVETRAN_.*_STAGING\..*", # fivetran r".*__DBT_TMP$", # dbt rf".*\.SEGMENT_{UUID_REGEX}", # segment rf".*\.STAGING_.*_{UUID_REGEX}", # stitch + r".*\.(GE_TMP_|GE_TEMP_|GX_TEMP_)[0-9A-F]{8}", # great expectations ] @@ -73,6 +81,93 @@ def source_database(self) -> DatabaseId: return DatabaseId(self.database, self.platform_instance) +class SnowflakeFilterConfig(SQLFilterConfig): + database_pattern: AllowDenyPattern = Field( + AllowDenyPattern( + deny=[r"^UTIL_DB$", r"^SNOWFLAKE$", r"^SNOWFLAKE_SAMPLE_DATA$"], + ), + description="Regex patterns for databases to filter in ingestion.", + ) + + schema_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns for schemas to filter in ingestion. Will match against the full `database.schema` name if `match_fully_qualified_names` is enabled.", + ) + # table_pattern and view_pattern are inherited from SQLFilterConfig + + match_fully_qualified_names: bool = Field( + default=False, + description="Whether `schema_pattern` is matched against fully qualified schema name `.`.", + ) + + @root_validator(pre=False, skip_on_failure=True) + def validate_legacy_schema_pattern(cls, values: Dict) -> Dict: + schema_pattern: Optional[AllowDenyPattern] = values.get("schema_pattern") + match_fully_qualified_names = values.get("match_fully_qualified_names") + + if ( + schema_pattern is not None + and schema_pattern != AllowDenyPattern.allow_all() + and match_fully_qualified_names is not None + and not match_fully_qualified_names + ): + logger.warning( + "Please update `schema_pattern` to match against fully qualified schema name `.` and set config `match_fully_qualified_names : True`." + "Current default `match_fully_qualified_names: False` is only to maintain backward compatibility. " + "The config option `match_fully_qualified_names` will be deprecated in future and the default behavior will assume `match_fully_qualified_names: True`." + ) + + # Always exclude reporting metadata for INFORMATION_SCHEMA schema + if schema_pattern is not None and schema_pattern: + logger.debug("Adding deny for INFORMATION_SCHEMA to schema_pattern.") + cast(AllowDenyPattern, schema_pattern).deny.append(r".*INFORMATION_SCHEMA$") + + return values + + +class SnowflakeIdentifierConfig( + PlatformInstanceConfigMixin, EnvConfigMixin, LowerCaseDatasetUrnConfigMixin +): + # Changing default value here. + convert_urns_to_lowercase: bool = Field( + default=True, + ) + + +# TODO: SnowflakeConfig is unused except for this inheritance. We should collapse the config inheritance hierarchy. +class SnowflakeConfig( + SnowflakeIdentifierConfig, + SnowflakeFilterConfig, + # SnowflakeFilterConfig must come before (higher precedence) the SQLCommon config, so that the documentation overrides are applied. + SnowflakeConnectionConfig, + BaseTimeWindowConfig, + SQLCommonConfig, +): + include_table_lineage: bool = pydantic.Field( + default=True, + description="If enabled, populates the snowflake table-to-table and s3-to-snowflake table lineage. Requires appropriate grants given to the role and Snowflake Enterprise Edition or above.", + ) + include_view_lineage: bool = pydantic.Field( + default=True, + description="If enabled, populates the snowflake view->table and table->view lineages. Requires appropriate grants given to the role, and include_table_lineage to be True. view->table lineage requires Snowflake Enterprise Edition or above.", + ) + + ignore_start_time_lineage: bool = False + upstream_lineage_in_report: bool = False + + @pydantic.root_validator(skip_on_failure=True) + def validate_include_view_lineage(cls, values): + if ( + "include_table_lineage" in values + and not values.get("include_table_lineage") + and values.get("include_view_lineage") + ): + raise ValueError( + "include_table_lineage must be True for include_view_lineage to be set." + ) + return values + + class SnowflakeV2Config( SnowflakeConfig, SnowflakeUsageConfig, @@ -81,10 +176,6 @@ class SnowflakeV2Config( StatefulProfilingConfigMixin, ClassificationSourceConfigMixin, ): - convert_urns_to_lowercase: bool = Field( - default=True, - ) - include_usage_stats: bool = Field( default=True, description="If enabled, populates the snowflake usage statistics. Requires appropriate grants given to the role.", @@ -95,6 +186,15 @@ class SnowflakeV2Config( description="If enabled, populates the snowflake technical schema and descriptions.", ) + include_primary_keys: bool = Field( + default=True, + description="If enabled, populates the snowflake primary keys.", + ) + include_foreign_keys: bool = Field( + default=True, + description="If enabled, populates the snowflake foreign keys.", + ) + include_column_lineage: bool = Field( default=True, description="Populates table->table and view->table column lineage. Requires appropriate grants given to the role and the Snowflake Enterprise Edition or above.", @@ -105,6 +205,12 @@ class SnowflakeV2Config( description="Populates view->view and table->view column lineage using DataHub's sql parser.", ) + lazy_schema_resolver: bool = Field( + default=False, + description="If enabled, uses lazy schema resolver to resolve schemas for tables and views. " + "This is useful if you have a large number of schemas and want to avoid bulk fetching the schema for each table/view.", + ) + _check_role_grants_removed = pydantic_removed_field("check_role_grants") _provision_role_removed = pydantic_removed_field("provision_role") @@ -118,11 +224,6 @@ class SnowflakeV2Config( description="Whether to populate Snowsight url for Snowflake Objects", ) - match_fully_qualified_names: bool = Field( - default=False, - description="Whether `schema_pattern` is matched against fully qualified schema name `.`.", - ) - _use_legacy_lineage_method_removed = pydantic_removed_field( "use_legacy_lineage_method" ) @@ -139,7 +240,7 @@ class SnowflakeV2Config( # This is required since access_history table does not capture whether the table was temporary table. temporary_tables_pattern: List[str] = Field( - default=DEFAULT_TABLES_DENY_LIST, + default=DEFAULT_TEMP_TABLES_PATTERNS, description="[Advanced] Regex patterns for temporary tables to filter in lineage ingestion. Specify regex to " "match the entire table name in database.schema.table format. Defaults are to set in such a way " "to ignore the temporary staging tables created by known ETL tools.", @@ -164,6 +265,12 @@ class SnowflakeV2Config( "username.", ) + include_assertion_results: bool = Field( + default=False, + description="Whether to ingest assertion run results for assertions created using Datahub" + " assertions CLI in snowflake", + ) + @validator("convert_urns_to_lowercase") def validate_convert_urns_to_lowercase(cls, v): if not v: @@ -189,27 +296,6 @@ def validate_unsupported_configs(cls, values: Dict) -> Dict: "include_read_operational_stats is not supported. Set `include_read_operational_stats` to False.", ) - match_fully_qualified_names = values.get("match_fully_qualified_names") - - schema_pattern: Optional[AllowDenyPattern] = values.get("schema_pattern") - - if ( - schema_pattern is not None - and schema_pattern != AllowDenyPattern.allow_all() - and match_fully_qualified_names is not None - and not match_fully_qualified_names - ): - logger.warning( - "Please update `schema_pattern` to match against fully qualified schema name `.` and set config `match_fully_qualified_names : True`." - "Current default `match_fully_qualified_names: False` is only to maintain backward compatibility. " - "The config option `match_fully_qualified_names` will be deprecated in future and the default behavior will assume `match_fully_qualified_names: True`." - ) - - # Always exclude reporting metadata for INFORMATION_SCHEMA schema - if schema_pattern is not None and schema_pattern: - logger.debug("Adding deny for INFORMATION_SCHEMA to schema_pattern.") - cast(AllowDenyPattern, schema_pattern).deny.append(r".*INFORMATION_SCHEMA$") - include_technical_schema = values.get("include_technical_schema") include_profiles = ( values.get("profiling") is not None and values["profiling"].enabled @@ -238,7 +324,7 @@ def get_sql_alchemy_url( password: Optional[SecretStr] = None, role: Optional[str] = None, ) -> str: - return BaseSnowflakeConfig.get_sql_alchemy_url( + return SnowflakeConnectionConfig.get_sql_alchemy_url( self, database=database, username=username, password=password, role=role ) diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py similarity index 81% rename from metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py rename to metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py index 521e755b6a00c..e981ed3e2e665 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py @@ -5,6 +5,8 @@ import snowflake.connector from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization +from snowflake.connector import SnowflakeConnection as NativeSnowflakeConnection +from snowflake.connector.cursor import DictCursor from snowflake.connector.network import ( DEFAULT_AUTHENTICATOR, EXTERNAL_BROWSER_AUTHENTICATOR, @@ -12,38 +14,41 @@ OAUTH_AUTHENTICATOR, ) -from datahub.configuration.common import AllowDenyPattern, ConfigModel +from datahub.configuration.common import ConfigModel, ConfigurationError, MetaError from datahub.configuration.connection_resolver import auto_connection_resolver from datahub.configuration.oauth import OAuthConfiguration, OAuthIdentityProvider -from datahub.configuration.time_window_config import BaseTimeWindowConfig from datahub.configuration.validate_field_rename import pydantic_renamed_field from datahub.ingestion.source.snowflake.constants import ( CLIENT_PREFETCH_THREADS, CLIENT_SESSION_KEEP_ALIVE, ) from datahub.ingestion.source.sql.oauth_generator import OAuthTokenGenerator -from datahub.ingestion.source.sql.sql_config import SQLCommonConfig, make_sqlalchemy_uri +from datahub.ingestion.source.sql.sql_config import make_sqlalchemy_uri from datahub.utilities.config_clean import ( remove_protocol, remove_suffix, remove_trailing_slashes, ) -logger: logging.Logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) -APPLICATION_NAME: str = "acryl_datahub" +_APPLICATION_NAME: str = "acryl_datahub" -VALID_AUTH_TYPES: Dict[str, str] = { +_VALID_AUTH_TYPES: Dict[str, str] = { "DEFAULT_AUTHENTICATOR": DEFAULT_AUTHENTICATOR, "EXTERNAL_BROWSER_AUTHENTICATOR": EXTERNAL_BROWSER_AUTHENTICATOR, "KEY_PAIR_AUTHENTICATOR": KEY_PAIR_AUTHENTICATOR, "OAUTH_AUTHENTICATOR": OAUTH_AUTHENTICATOR, } -SNOWFLAKE_HOST_SUFFIX = ".snowflakecomputing.com" +_SNOWFLAKE_HOST_SUFFIX = ".snowflakecomputing.com" -class BaseSnowflakeConfig(ConfigModel): +class SnowflakePermissionError(MetaError): + """A permission error has happened""" + + +class SnowflakeConnectionConfig(ConfigModel): # Note: this config model is also used by the snowflake-usage source. _connection = auto_connection_resolver() @@ -106,15 +111,15 @@ def get_account(self) -> str: def validate_account_id(cls, account_id: str) -> str: account_id = remove_protocol(account_id) account_id = remove_trailing_slashes(account_id) - account_id = remove_suffix(account_id, SNOWFLAKE_HOST_SUFFIX) + account_id = remove_suffix(account_id, _SNOWFLAKE_HOST_SUFFIX) return account_id @pydantic.validator("authentication_type", always=True) def authenticator_type_is_valid(cls, v, values): - if v not in VALID_AUTH_TYPES.keys(): + if v not in _VALID_AUTH_TYPES.keys(): raise ValueError( f"unsupported authenticator type '{v}' was provided," - f" use one of {list(VALID_AUTH_TYPES.keys())}" + f" use one of {list(_VALID_AUTH_TYPES.keys())}" ) if ( values.get("private_key") is not None @@ -189,10 +194,10 @@ def get_sql_alchemy_url( # Drop the options if value is None. key: value for (key, value) in { - "authenticator": VALID_AUTH_TYPES.get(self.authentication_type), + "authenticator": _VALID_AUTH_TYPES.get(self.authentication_type), "warehouse": self.warehouse, "role": role, - "application": APPLICATION_NAME, + "application": _APPLICATION_NAME, }.items() if value }, @@ -255,7 +260,7 @@ def get_options(self) -> dict: self.options["connect_args"] = options_connect_args return self.options - def get_oauth_connection(self) -> snowflake.connector.SnowflakeConnection: + def get_oauth_connection(self) -> NativeSnowflakeConnection: assert ( self.oauth_config ), "oauth_config should be provided if using oauth based authentication" @@ -292,12 +297,12 @@ def get_oauth_connection(self) -> snowflake.connector.SnowflakeConnection: token=token, role=self.role, warehouse=self.warehouse, - authenticator=VALID_AUTH_TYPES.get(self.authentication_type), - application=APPLICATION_NAME, + authenticator=_VALID_AUTH_TYPES.get(self.authentication_type), + application=_APPLICATION_NAME, **connect_args, ) - def get_key_pair_connection(self) -> snowflake.connector.SnowflakeConnection: + def get_key_pair_connection(self) -> NativeSnowflakeConnection: connect_args = self.get_options()["connect_args"] return snowflake.connector.connect( @@ -305,12 +310,12 @@ def get_key_pair_connection(self) -> snowflake.connector.SnowflakeConnection: account=self.account_id, warehouse=self.warehouse, role=self.role, - authenticator=VALID_AUTH_TYPES.get(self.authentication_type), - application=APPLICATION_NAME, + authenticator=_VALID_AUTH_TYPES.get(self.authentication_type), + application=_APPLICATION_NAME, **connect_args, ) - def get_connection(self) -> snowflake.connector.SnowflakeConnection: + def get_native_connection(self) -> NativeSnowflakeConnection: connect_args = self.get_options()["connect_args"] if self.authentication_type == "DEFAULT_AUTHENTICATOR": return snowflake.connector.connect( @@ -319,7 +324,7 @@ def get_connection(self) -> snowflake.connector.SnowflakeConnection: account=self.account_id, warehouse=self.warehouse, role=self.role, - application=APPLICATION_NAME, + application=_APPLICATION_NAME, **connect_args, ) elif self.authentication_type == "OAUTH_AUTHENTICATOR": @@ -333,40 +338,59 @@ def get_connection(self) -> snowflake.connector.SnowflakeConnection: account=self.account_id, warehouse=self.warehouse, role=self.role, - authenticator=VALID_AUTH_TYPES.get(self.authentication_type), - application=APPLICATION_NAME, + authenticator=_VALID_AUTH_TYPES.get(self.authentication_type), + application=_APPLICATION_NAME, **connect_args, ) else: # not expected to be here raise Exception("Not expected to be here.") + def get_connection(self) -> "SnowflakeConnection": + try: + return SnowflakeConnection(self.get_native_connection()) + except Exception as e: + logger.debug(e, exc_info=e) -class SnowflakeConfig(BaseSnowflakeConfig, BaseTimeWindowConfig, SQLCommonConfig): - include_table_lineage: bool = pydantic.Field( - default=True, - description="If enabled, populates the snowflake table-to-table and s3-to-snowflake table lineage. Requires appropriate grants given to the role and Snowflake Enterprise Edition or above.", - ) - include_view_lineage: bool = pydantic.Field( - default=True, - description="If enabled, populates the snowflake view->table and table->view lineages. Requires appropriate grants given to the role, and include_table_lineage to be True. view->table lineage requires Snowflake Enterprise Edition or above.", - ) + if "not granted to this user" in str(e): + raise SnowflakePermissionError( + f"Permissions error when connecting to snowflake: {e}" + ) from e - database_pattern: AllowDenyPattern = AllowDenyPattern( - deny=[r"^UTIL_DB$", r"^SNOWFLAKE$", r"^SNOWFLAKE_SAMPLE_DATA$"] - ) + raise ConfigurationError( + f"Failed to connect to snowflake instance: {e}" + ) from e - ignore_start_time_lineage: bool = False - upstream_lineage_in_report: bool = False - @pydantic.root_validator(skip_on_failure=True) - def validate_include_view_lineage(cls, values): - if ( - "include_table_lineage" in values - and not values.get("include_table_lineage") - and values.get("include_view_lineage") - ): - raise ValueError( - "include_table_lineage must be True for include_view_lineage to be set." - ) - return values +class SnowflakeConnection: + _connection: NativeSnowflakeConnection + + def __init__(self, connection: NativeSnowflakeConnection): + self._connection = connection + + def native_connection(self) -> NativeSnowflakeConnection: + return self._connection + + def query(self, query: str) -> Any: + try: + logger.info(f"Query: {query}", stacklevel=2) + resp = self._connection.cursor(DictCursor).execute(query) + return resp + + except Exception as e: + if _is_permission_error(e): + raise SnowflakePermissionError(e) from e + raise + + def is_closed(self) -> bool: + return self._connection.is_closed() + + def close(self): + self._connection.close() + + +def _is_permission_error(e: Exception) -> bool: + msg = str(e) + # 002003 (02000): SQL compilation error: Database/SCHEMA 'XXXX' does not exist or not authorized. + # Insufficient privileges to operate on database 'XXXX' + return "Insufficient privileges" in msg or "not authorized" in msg diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_data_reader.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_data_reader.py index 9fa81cb1bd20c..c9615bb498fe4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_data_reader.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_data_reader.py @@ -2,9 +2,9 @@ from typing import Any, Callable, Dict, List import pandas as pd -from snowflake.connector import SnowflakeConnection from datahub.ingestion.source.common.data_reader import DataReader +from datahub.ingestion.source.snowflake.snowflake_connection import SnowflakeConnection from datahub.utilities.perf_timer import PerfTimer logger = logging.Logger(__name__) @@ -39,7 +39,7 @@ def get_sample_data_for_table( logger.debug( f"Collecting sample values for table {db_name}.{schema_name}.{table_name}" ) - with PerfTimer() as timer, self.conn.cursor() as cursor: + with PerfTimer() as timer, self.conn.native_connection().cursor() as cursor: sql = f'select * from "{db_name}"."{schema_name}"."{table_name}" sample ({sample_size} rows);' cursor.execute(sql) dat = cursor.fetchall() diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py index b12ef4d19c45c..3e65f06200418 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py @@ -5,9 +5,9 @@ from typing import Any, Callable, Collection, Iterable, List, Optional, Set, Tuple, Type from pydantic import BaseModel, validator -from snowflake.connector import SnowflakeConnection from datahub.configuration.datetimes import parse_absolute_time +from datahub.ingestion.api.closeable import Closeable from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.aws.s3_util import make_s3_urn_for_lineage from datahub.ingestion.source.snowflake.constants import ( @@ -15,14 +15,13 @@ SnowflakeEdition, ) from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeV2Config -from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery -from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report -from datahub.ingestion.source.snowflake.snowflake_utils import ( - SnowflakeCommonMixin, - SnowflakeConnectionMixin, +from datahub.ingestion.source.snowflake.snowflake_connection import ( + SnowflakeConnection, SnowflakePermissionError, - SnowflakeQueryMixin, ) +from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery +from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report +from datahub.ingestion.source.snowflake.snowflake_utils import SnowflakeCommonMixin from datahub.ingestion.source.state.redundant_run_skip_handler import ( RedundantLineageRunSkipHandler, ) @@ -30,6 +29,7 @@ from datahub.sql_parsing.sql_parsing_aggregator import ( ColumnLineageInfo, ColumnRef, + KnownLineageMapping, KnownQueryLineageInfo, SqlParsingAggregator, UrnStr, @@ -101,9 +101,7 @@ class SnowflakeColumnId: object_domain: Optional[str] = None -class SnowflakeLineageExtractor( - SnowflakeQueryMixin, SnowflakeConnectionMixin, SnowflakeCommonMixin -): +class SnowflakeLineageExtractor(SnowflakeCommonMixin, Closeable): """ Extracts Lineage from Snowflake. Following lineage edges are considered. @@ -120,6 +118,7 @@ def __init__( self, config: SnowflakeV2Config, report: SnowflakeV2Report, + connection: SnowflakeConnection, dataset_urn_builder: Callable[[str], str], redundant_run_skip_handler: Optional[RedundantLineageRunSkipHandler], sql_aggregator: SqlParsingAggregator, @@ -128,7 +127,7 @@ def __init__( self.report = report self.logger = logger self.dataset_urn_builder = dataset_urn_builder - self.connection: Optional[SnowflakeConnection] = None + self.connection = connection self.sql_aggregator = sql_aggregator self.redundant_run_skip_handler = redundant_run_skip_handler @@ -165,10 +164,6 @@ def get_workunits( if not self._should_ingest_lineage(): return - self.connection = self.create_connection() - if self.connection is None: - return - # s3 dataset -> snowflake table self._populate_external_upstreams(discovered_tables) @@ -264,13 +259,20 @@ def _populate_external_upstreams(self, discovered_tables: List[str]) -> None: with PerfTimer() as timer: self.report.num_external_table_edges_scanned = 0 - self._populate_external_lineage_from_copy_history(discovered_tables) + for ( + known_lineage_mapping + ) in self._populate_external_lineage_from_copy_history(discovered_tables): + self.sql_aggregator.add(known_lineage_mapping) logger.info( "Done populating external lineage from copy history. " f"Found {self.report.num_external_table_edges_scanned} external lineage edges so far." ) - self._populate_external_lineage_from_show_query(discovered_tables) + for ( + known_lineage_mapping + ) in self._populate_external_lineage_from_show_query(discovered_tables): + self.sql_aggregator.add(known_lineage_mapping) + logger.info( "Done populating external lineage from show external tables. " f"Found {self.report.num_external_table_edges_scanned} external lineage edges so far." @@ -282,10 +284,10 @@ def _populate_external_upstreams(self, discovered_tables: List[str]) -> None: # NOTE: Snowflake does not log this information to the access_history table. def _populate_external_lineage_from_show_query( self, discovered_tables: List[str] - ) -> None: + ) -> Iterable[KnownLineageMapping]: external_tables_query: str = SnowflakeQuery.show_external_tables() try: - for db_row in self.query(external_tables_query): + for db_row in self.connection.query(external_tables_query): key = self.get_dataset_identifier( db_row["name"], db_row["schema_name"], db_row["database_name"] ) @@ -293,11 +295,11 @@ def _populate_external_lineage_from_show_query( if key not in discovered_tables: continue if db_row["location"].startswith("s3://"): - self.sql_aggregator.add_known_lineage_mapping( - downstream_urn=self.dataset_urn_builder(key), + yield KnownLineageMapping( upstream_urn=make_s3_urn_for_lineage( db_row["location"], self.config.env ), + downstream_urn=self.dataset_urn_builder(key), ) self.report.num_external_table_edges_scanned += 1 @@ -316,7 +318,7 @@ def _populate_external_lineage_from_show_query( # NOTE: Snowflake does not log this information to the access_history table. def _populate_external_lineage_from_copy_history( self, discovered_tables: List[str] - ) -> None: + ) -> Iterable[KnownLineageMapping]: query: str = SnowflakeQuery.copy_lineage_history( start_time_millis=int(self.start_time.timestamp() * 1000), end_time_millis=int(self.end_time.timestamp() * 1000), @@ -324,8 +326,12 @@ def _populate_external_lineage_from_copy_history( ) try: - for db_row in self.query(query): - self._process_external_lineage_result_row(db_row, discovered_tables) + for db_row in self.connection.query(query): + known_lineage_mapping = self._process_external_lineage_result_row( + db_row, discovered_tables + ) + if known_lineage_mapping: + yield known_lineage_mapping except Exception as e: if isinstance(e, SnowflakePermissionError): error_msg = "Failed to get external lineage. Please grant imported privileges on SNOWFLAKE database. " @@ -340,35 +346,40 @@ def _populate_external_lineage_from_copy_history( def _process_external_lineage_result_row( self, db_row: dict, discovered_tables: List[str] - ) -> None: + ) -> Optional[KnownLineageMapping]: # key is the down-stream table name key: str = self.get_dataset_identifier_from_qualified_name( db_row["DOWNSTREAM_TABLE_NAME"] ) if key not in discovered_tables: - return + return None if db_row["UPSTREAM_LOCATIONS"] is not None: external_locations = json.loads(db_row["UPSTREAM_LOCATIONS"]) for loc in external_locations: if loc.startswith("s3://"): - self.sql_aggregator.add_known_lineage_mapping( - downstream_urn=self.dataset_urn_builder(key), + self.report.num_external_table_edges_scanned += 1 + return KnownLineageMapping( upstream_urn=make_s3_urn_for_lineage(loc, self.config.env), + downstream_urn=self.dataset_urn_builder(key), ) - self.report.num_external_table_edges_scanned += 1 + + return None def _fetch_upstream_lineages_for_tables(self) -> Iterable[UpstreamLineageEdge]: query: str = SnowflakeQuery.table_to_table_lineage_history_v2( start_time_millis=int(self.start_time.timestamp() * 1000), end_time_millis=int(self.end_time.timestamp() * 1000), upstreams_deny_pattern=self.config.temporary_tables_pattern, - include_view_lineage=self.config.include_view_lineage, + # The self.config.include_view_lineage setting is about fetching upstreams of views. + # We always generate lineage pointing at views from tables, even if self.config.include_view_lineage is False. + # TODO: Remove this `include_view_lineage` flag, since it's effectively dead code. + include_view_lineage=True, include_column_lineage=self.config.include_column_lineage, ) try: - for db_row in self.query(query): + for db_row in self.connection.query(query): edge = self._process_upstream_lineage_row(db_row) if edge: yield edge @@ -409,10 +420,12 @@ def map_query_result_upstreams( upstream_name = self.get_dataset_identifier_from_qualified_name( upstream_table.upstream_object_name ) - if upstream_name and self._is_dataset_pattern_allowed( - upstream_name, - upstream_table.upstream_object_domain, - is_upstream=True, + if upstream_name and ( + not self.config.validate_upstreams_against_patterns + or self.is_dataset_pattern_allowed( + upstream_name, + upstream_table.upstream_object_domain, + ) ): upstreams.append(self.dataset_urn_builder(upstream_name)) except Exception as e: @@ -493,10 +506,12 @@ def build_finegrained_lineage_upstreams( if ( upstream_col.object_name and upstream_col.column_name - and self._is_dataset_pattern_allowed( - upstream_col.object_name, - upstream_col.object_domain, - is_upstream=True, + and ( + not self.config.validate_upstreams_against_patterns + or self.is_dataset_pattern_allowed( + upstream_col.object_name, + upstream_col.object_domain, + ) ) ): upstream_dataset_name = self.get_dataset_identifier_from_qualified_name( @@ -547,3 +562,6 @@ def _should_ingest_lineage(self) -> bool: def report_status(self, step: str, status: bool) -> None: if self.redundant_run_skip_handler: self.redundant_run_skip_handler.report_current_run_status(step, status) + + def close(self) -> None: + pass diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py index 5e6ade29344eb..4deeb9f96f48e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py @@ -164,7 +164,7 @@ def callable_for_db_connection(self, db_name: str) -> Callable: schema_name = self.database_default_schema.get(db_name) def get_db_connection(): - conn = self.config.get_connection() + conn = self.config.get_native_connection() conn.cursor().execute(SnowflakeQuery.use_database(db_name)) # As mentioned here - https://docs.snowflake.com/en/sql-reference/sql/use-database#usage-notes diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py new file mode 100644 index 0000000000000..c647a624a5467 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py @@ -0,0 +1,515 @@ +import functools +import json +import logging +import pathlib +import re +import tempfile +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Dict, Iterable, List, Optional, Union + +import pydantic +from typing_extensions import Self + +from datahub.configuration.time_window_config import ( + BaseTimeWindowConfig, + BucketDuration, +) +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.api.report import Report +from datahub.ingestion.api.source import Source, SourceReport +from datahub.ingestion.api.source_helpers import auto_workunit +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.snowflake.constants import SnowflakeObjectDomain +from datahub.ingestion.source.snowflake.snowflake_config import ( + DEFAULT_TEMP_TABLES_PATTERNS, + SnowflakeFilterConfig, + SnowflakeIdentifierConfig, +) +from datahub.ingestion.source.snowflake.snowflake_connection import ( + SnowflakeConnection, + SnowflakeConnectionConfig, +) +from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery +from datahub.ingestion.source.snowflake.snowflake_utils import ( + SnowflakeFilterMixin, + SnowflakeIdentifierMixin, +) +from datahub.ingestion.source.usage.usage_common import BaseUsageConfig +from datahub.metadata.urns import CorpUserUrn +from datahub.sql_parsing.sql_parsing_aggregator import ( + KnownLineageMapping, + PreparsedQuery, + SqlAggregatorReport, + SqlParsingAggregator, +) +from datahub.sql_parsing.sql_parsing_common import QueryType +from datahub.sql_parsing.sqlglot_lineage import ( + ColumnLineageInfo, + ColumnRef, + DownstreamColumnRef, +) +from datahub.utilities.file_backed_collections import ConnectionWrapper, FileBackedList + +logger = logging.getLogger(__name__) + + +class SnowflakeQueriesExtractorConfig(SnowflakeIdentifierConfig, SnowflakeFilterConfig): + # TODO: Support stateful ingestion for the time windows. + window: BaseTimeWindowConfig = BaseTimeWindowConfig() + + # TODO: make this a proper allow/deny pattern + deny_usernames: List[str] = [] + + temporary_tables_pattern: List[str] = pydantic.Field( + default=DEFAULT_TEMP_TABLES_PATTERNS, + description="[Advanced] Regex patterns for temporary tables to filter in lineage ingestion. Specify regex to " + "match the entire table name in database.schema.table format. Defaults are to set in such a way " + "to ignore the temporary staging tables created by known ETL tools.", + ) + + local_temp_path: Optional[pathlib.Path] = pydantic.Field( + default=None, + description="Local path to store the audit log.", + # TODO: For now, this is simply an advanced config to make local testing easier. + # Eventually, we will want to store date-specific files in the directory and use it as a cache. + hidden_from_docs=True, + ) + + convert_urns_to_lowercase: bool = pydantic.Field( + # Override the default. + default=True, + description="Whether to convert dataset urns to lowercase.", + ) + + include_lineage: bool = True + include_queries: bool = True + include_usage_statistics: bool = True + include_query_usage_statistics: bool = False + include_operations: bool = True + + +class SnowflakeQueriesSourceConfig(SnowflakeQueriesExtractorConfig): + connection: SnowflakeConnectionConfig + + +@dataclass +class SnowflakeQueriesExtractorReport(Report): + window: Optional[BaseTimeWindowConfig] = None + + sql_aggregator: Optional[SqlAggregatorReport] = None + + +@dataclass +class SnowflakeQueriesSourceReport(SourceReport): + queries_extractor: Optional[SnowflakeQueriesExtractorReport] = None + + +class SnowflakeQueriesExtractor(SnowflakeFilterMixin, SnowflakeIdentifierMixin): + def __init__( + self, + connection: SnowflakeConnection, + config: SnowflakeQueriesExtractorConfig, + structured_report: SourceReport, + ): + self.connection = connection + + self.config = config + self.report = SnowflakeQueriesExtractorReport() + self._structured_report = structured_report + + self.aggregator = SqlParsingAggregator( + platform=self.platform, + platform_instance=self.config.platform_instance, + env=self.config.env, + # graph=self.ctx.graph, + generate_lineage=self.config.include_lineage, + generate_queries=self.config.include_queries, + generate_usage_statistics=self.config.include_usage_statistics, + generate_query_usage_statistics=self.config.include_query_usage_statistics, + usage_config=BaseUsageConfig( + bucket_duration=self.config.window.bucket_duration, + start_time=self.config.window.start_time, + end_time=self.config.window.end_time, + # TODO make the rest of the fields configurable + ), + generate_operations=self.config.include_operations, + is_temp_table=self.is_temp_table, + is_allowed_table=self.is_allowed_table, + format_queries=False, + ) + self.report.sql_aggregator = self.aggregator.report + + @property + def structured_reporter(self) -> SourceReport: + return self._structured_report + + @property + def filter_config(self) -> SnowflakeFilterConfig: + return self.config + + @property + def identifier_config(self) -> SnowflakeIdentifierConfig: + return self.config + + @functools.cached_property + def local_temp_path(self) -> pathlib.Path: + if self.config.local_temp_path: + assert self.config.local_temp_path.is_dir() + return self.config.local_temp_path + + path = pathlib.Path(tempfile.mkdtemp()) + path.mkdir(parents=True, exist_ok=True) + logger.info(f"Using local temp path: {path}") + return path + + def is_temp_table(self, name: str) -> bool: + return any( + re.match(pattern, name, flags=re.IGNORECASE) + for pattern in self.config.temporary_tables_pattern + ) + + def is_allowed_table(self, name: str) -> bool: + return self.is_dataset_pattern_allowed(name, SnowflakeObjectDomain.TABLE) + + def get_workunits_internal( + self, + ) -> Iterable[MetadataWorkUnit]: + self.report.window = self.config.window + + # TODO: Add some logic to check if the cached audit log is stale or not. + audit_log_file = self.local_temp_path / "audit_log.sqlite" + use_cached_audit_log = audit_log_file.exists() + + queries: FileBackedList[Union[KnownLineageMapping, PreparsedQuery]] + if use_cached_audit_log: + logger.info("Using cached audit log") + shared_connection = ConnectionWrapper(audit_log_file) + queries = FileBackedList(shared_connection) + else: + audit_log_file.unlink(missing_ok=True) + + shared_connection = ConnectionWrapper(audit_log_file) + queries = FileBackedList(shared_connection) + + logger.info("Fetching audit log") + for entry in self.fetch_audit_log(): + queries.append(entry) + + for query in queries: + self.aggregator.add(query) + + yield from auto_workunit(self.aggregator.gen_metadata()) + + def fetch_audit_log( + self, + ) -> Iterable[Union[KnownLineageMapping, PreparsedQuery]]: + """ + # TODO: we need to fetch this info from somewhere + discovered_tables = [] + + snowflake_lineage_v2 = SnowflakeLineageExtractor( + config=self.config, # type: ignore + report=self.report, # type: ignore + dataset_urn_builder=self.gen_dataset_urn, + redundant_run_skip_handler=None, + sql_aggregator=self.aggregator, # TODO this should be unused + ) + + for ( + known_lineage_mapping + ) in snowflake_lineage_v2._populate_external_lineage_from_copy_history( + discovered_tables=discovered_tables + ): + interim_results.append(known_lineage_mapping) + + for ( + known_lineage_mapping + ) in snowflake_lineage_v2._populate_external_lineage_from_show_query( + discovered_tables=discovered_tables + ): + interim_results.append(known_lineage_mapping) + """ + + audit_log_query = _build_enriched_audit_log_query( + start_time=self.config.window.start_time, + end_time=self.config.window.end_time, + bucket_duration=self.config.window.bucket_duration, + deny_usernames=self.config.deny_usernames, + ) + + resp = self.connection.query(audit_log_query) + + for i, row in enumerate(resp): + if i % 1000 == 0: + logger.info(f"Processed {i} audit log rows") + + assert isinstance(row, dict) + try: + entry = self._parse_audit_log_row(row) + except Exception as e: + self.structured_reporter.warning( + "Error parsing audit log row", + context=f"{row}", + exc=e, + ) + else: + yield entry + + def get_dataset_identifier_from_qualified_name(self, qualified_name: str) -> str: + # Copied from SnowflakeCommonMixin. + return self.snowflake_identifier(self.cleanup_qualified_name(qualified_name)) + + def _parse_audit_log_row(self, row: Dict[str, Any]) -> PreparsedQuery: + json_fields = { + "DIRECT_OBJECTS_ACCESSED", + "OBJECTS_MODIFIED", + } + + res = {} + for key, value in row.items(): + if key in json_fields and value: + value = json.loads(value) + key = key.lower() + res[key] = value + + direct_objects_accessed = res["direct_objects_accessed"] + objects_modified = res["objects_modified"] + + upstreams = [] + column_usage = {} + + for obj in direct_objects_accessed: + dataset = self.gen_dataset_urn( + self.get_dataset_identifier_from_qualified_name(obj["objectName"]) + ) + + columns = set() + for modified_column in obj["columns"]: + columns.add(self.snowflake_identifier(modified_column["columnName"])) + + upstreams.append(dataset) + column_usage[dataset] = columns + + downstream = None + column_lineage = None + for obj in objects_modified: + # We don't expect there to be more than one object modified. + if downstream: + self.structured_reporter.report_warning( + message="Unexpectedly got multiple downstream entities from the Snowflake audit log.", + context=f"{row}", + ) + + downstream = self.gen_dataset_urn( + self.get_dataset_identifier_from_qualified_name(obj["objectName"]) + ) + column_lineage = [] + for modified_column in obj["columns"]: + column_lineage.append( + ColumnLineageInfo( + downstream=DownstreamColumnRef( + dataset=downstream, + column=self.snowflake_identifier( + modified_column["columnName"] + ), + ), + upstreams=[ + ColumnRef( + table=self.gen_dataset_urn( + self.get_dataset_identifier_from_qualified_name( + upstream["objectName"] + ) + ), + column=self.snowflake_identifier( + upstream["columnName"] + ), + ) + for upstream in modified_column["directSources"] + if upstream["objectDomain"] + in SnowflakeQuery.ACCESS_HISTORY_TABLE_VIEW_DOMAINS + ], + ) + ) + + # TODO: Support filtering the table names. + # if objects_modified: + # breakpoint() + + # TODO implement email address mapping + user = CorpUserUrn(res["user_name"]) + + timestamp: datetime = res["query_start_time"] + timestamp = timestamp.astimezone(timezone.utc) + + # TODO need to map snowflake query types to ours + query_type = SNOWFLAKE_QUERY_TYPE_MAPPING.get( + res["query_type"], QueryType.UNKNOWN + ) + + entry = PreparsedQuery( + query_id=res["query_fingerprint"], + query_text=res["query_text"], + upstreams=upstreams, + downstream=downstream, + column_lineage=column_lineage, + column_usage=column_usage, + inferred_schema=None, + confidence_score=1, + query_count=res["query_count"], + user=user, + timestamp=timestamp, + session_id=res["session_id"], + query_type=query_type, + ) + return entry + + +class SnowflakeQueriesSource(Source): + def __init__(self, ctx: PipelineContext, config: SnowflakeQueriesSourceConfig): + self.ctx = ctx + self.config = config + self.report = SnowflakeQueriesSourceReport() + + self.platform = "snowflake" + + self.connection = self.config.connection.get_connection() + + self.queries_extractor = SnowflakeQueriesExtractor( + connection=self.connection, + config=self.config, + structured_report=self.report, + ) + self.report.queries_extractor = self.queries_extractor.report + + @classmethod + def create(cls, config_dict: dict, ctx: PipelineContext) -> Self: + config = SnowflakeQueriesSourceConfig.parse_obj(config_dict) + return cls(ctx, config) + + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: + # TODO: Disable auto status processor? + return self.queries_extractor.get_workunits_internal() + + def get_report(self) -> SnowflakeQueriesSourceReport: + return self.report + + +# Make sure we don't try to generate too much info for a single query. +_MAX_TABLES_PER_QUERY = 20 + + +def _build_enriched_audit_log_query( + start_time: datetime, + end_time: datetime, + bucket_duration: BucketDuration, + deny_usernames: Optional[List[str]], +) -> str: + start_time_millis = int(start_time.timestamp() * 1000) + end_time_millis = int(end_time.timestamp() * 1000) + + users_filter = "" + if deny_usernames: + user_not_in = ",".join(f"'{user.upper()}'" for user in deny_usernames) + users_filter = f"user_name NOT IN ({user_not_in})" + + time_bucket_size = bucket_duration.value + assert time_bucket_size in ("HOUR", "DAY", "MONTH") + + return f"""\ +WITH +fingerprinted_queries as ( + SELECT *, + -- TODO: Generate better fingerprints for each query by pushing down regex logic. + query_history.query_parameterized_hash as query_fingerprint + FROM + snowflake.account_usage.query_history + WHERE + query_history.start_time >= to_timestamp_ltz({start_time_millis}, 3) + AND query_history.start_time < to_timestamp_ltz({end_time_millis}, 3) + AND execution_status = 'SUCCESS' + AND {users_filter or 'TRUE'} +) +, deduplicated_queries as ( + SELECT + *, + DATE_TRUNC( + {time_bucket_size}, + CONVERT_TIMEZONE('UTC', start_time) + ) AS bucket_start_time, + COUNT(*) OVER (PARTITION BY bucket_start_time, query_fingerprint) AS query_count, + FROM + fingerprinted_queries + QUALIFY + ROW_NUMBER() OVER (PARTITION BY bucket_start_time, query_fingerprint ORDER BY start_time DESC) = 1 +) +, raw_access_history AS ( + SELECT + query_id, + query_start_time, + user_name, + direct_objects_accessed, + objects_modified, + FROM + snowflake.account_usage.access_history + WHERE + query_start_time >= to_timestamp_ltz({start_time_millis}, 3) + AND query_start_time < to_timestamp_ltz({end_time_millis}, 3) + AND {users_filter or 'TRUE'} + AND query_id IN ( + SELECT query_id FROM deduplicated_queries + ) +) +, filtered_access_history AS ( + -- TODO: Add table filter clause. + SELECT + query_id, + query_start_time, + ARRAY_SLICE( + FILTER(direct_objects_accessed, o -> o:objectDomain IN {SnowflakeQuery.ACCESS_HISTORY_TABLE_VIEW_DOMAINS_FILTER}), + 0, {_MAX_TABLES_PER_QUERY} + ) as direct_objects_accessed, + -- TODO: Drop the columns.baseSources subfield. + FILTER(objects_modified, o -> o:objectDomain IN {SnowflakeQuery.ACCESS_HISTORY_TABLE_VIEW_DOMAINS_FILTER}) as objects_modified, + FROM raw_access_history + WHERE ( array_size(direct_objects_accessed) > 0 or array_size(objects_modified) > 0 ) +) +, query_access_history AS ( + SELECT + q.bucket_start_time, + q.query_id, + q.query_fingerprint, + q.query_count, + q.session_id AS "SESSION_ID", + q.start_time AS "QUERY_START_TIME", + q.total_elapsed_time AS "QUERY_DURATION", + q.query_text AS "QUERY_TEXT", + q.query_type AS "QUERY_TYPE", + q.database_name as "DEFAULT_DB", + q.schema_name as "DEFAULT_SCHEMA", + q.rows_inserted AS "ROWS_INSERTED", + q.rows_updated AS "ROWS_UPDATED", + q.rows_deleted AS "ROWS_DELETED", + q.user_name AS "USER_NAME", + q.role_name AS "ROLE_NAME", + a.direct_objects_accessed, + a.objects_modified, + FROM deduplicated_queries q + JOIN filtered_access_history a USING (query_id) +) +SELECT * FROM query_access_history +""" + + +SNOWFLAKE_QUERY_TYPE_MAPPING = { + "INSERT": QueryType.INSERT, + "UPDATE": QueryType.UPDATE, + "DELETE": QueryType.DELETE, + "CREATE": QueryType.CREATE_OTHER, + "CREATE_TABLE": QueryType.CREATE_DDL, + "CREATE_VIEW": QueryType.CREATE_VIEW, + "CREATE_TABLE_AS_SELECT": QueryType.CREATE_TABLE_AS_SELECT, + "MERGE": QueryType.MERGE, + "COPY": QueryType.UNKNOWN, + "TRUNCATE_TABLE": QueryType.UNKNOWN, +} diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py index dac43499a1c71..a2e18a64d9a80 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py @@ -3,7 +3,12 @@ from datahub.configuration.common import AllowDenyPattern from datahub.configuration.time_window_config import BucketDuration from datahub.ingestion.source.snowflake.constants import SnowflakeObjectDomain -from datahub.ingestion.source.snowflake.snowflake_config import DEFAULT_TABLES_DENY_LIST +from datahub.ingestion.source.snowflake.snowflake_config import ( + DEFAULT_TEMP_TABLES_PATTERNS, +) +from datahub.utilities.prefix_batch_builder import PrefixGroup + +SHOW_VIEWS_MAX_PAGE_SIZE = 10000 def create_deny_regex_sql_filter( @@ -25,13 +30,15 @@ def create_deny_regex_sql_filter( class SnowflakeQuery: - ACCESS_HISTORY_TABLE_VIEW_DOMAINS_FILTER = ( - "(" - f"'{SnowflakeObjectDomain.TABLE.capitalize()}'," - f"'{SnowflakeObjectDomain.EXTERNAL_TABLE.capitalize()}'," - f"'{SnowflakeObjectDomain.VIEW.capitalize()}'," - f"'{SnowflakeObjectDomain.MATERIALIZED_VIEW.capitalize()}'" - ")" + ACCESS_HISTORY_TABLE_VIEW_DOMAINS = { + SnowflakeObjectDomain.TABLE.capitalize(), + SnowflakeObjectDomain.EXTERNAL_TABLE.capitalize(), + SnowflakeObjectDomain.VIEW.capitalize(), + SnowflakeObjectDomain.MATERIALIZED_VIEW.capitalize(), + } + + ACCESS_HISTORY_TABLE_VIEW_DOMAINS_FILTER = "({})".format( + ",".join(f"'{domain}'" for domain in ACCESS_HISTORY_TABLE_VIEW_DOMAINS) ) ACCESS_HISTORY_TABLE_DOMAINS_FILTER = ( "(" @@ -202,94 +209,74 @@ def get_tags_on_columns_with_propagation( FROM table("{db_name}".information_schema.tag_references_all_columns('{quoted_table_identifier}', '{SnowflakeObjectDomain.TABLE}')); """ - # View definition is retrived in information_schema query only if role is owner of view. Hence this query is not used. - # https://community.snowflake.com/s/article/Is-it-possible-to-see-the-view-definition-in-information-schema-views-from-a-non-owner-role - @staticmethod - def views_for_database(db_name: Optional[str]) -> str: - db_clause = f'"{db_name}".' if db_name is not None else "" - return f""" - SELECT table_catalog AS "TABLE_CATALOG", - table_schema AS "TABLE_SCHEMA", - table_name AS "TABLE_NAME", - created AS "CREATED", - last_altered AS "LAST_ALTERED", - comment AS "COMMENT", - view_definition AS "VIEW_DEFINITION" - FROM {db_clause}information_schema.views t - WHERE table_schema != 'INFORMATION_SCHEMA' - order by table_schema, table_name""" - - # View definition is retrived in information_schema query only if role is owner of view. Hence this query is not used. - # https://community.snowflake.com/s/article/Is-it-possible-to-see-the-view-definition-in-information-schema-views-from-a-non-owner-role @staticmethod - def views_for_schema(schema_name: str, db_name: Optional[str]) -> str: - db_clause = f'"{db_name}".' if db_name is not None else "" - return f""" - SELECT table_catalog AS "TABLE_CATALOG", - table_schema AS "TABLE_SCHEMA", - table_name AS "TABLE_NAME", - created AS "CREATED", - last_altered AS "LAST_ALTERED", - comment AS "COMMENT", - view_definition AS "VIEW_DEFINITION" - FROM {db_clause}information_schema.views t - where table_schema='{schema_name}' - order by table_schema, table_name""" + def show_views_for_database( + db_name: str, + limit: int = SHOW_VIEWS_MAX_PAGE_SIZE, + view_pagination_marker: Optional[str] = None, + ) -> str: + # While there is an information_schema.views view, that only shows the view definition if the role + # is an owner of the view. That doesn't work for us. + # https://community.snowflake.com/s/article/Is-it-possible-to-see-the-view-definition-in-information-schema-views-from-a-non-owner-role - @staticmethod - def show_views_for_database(db_name: str) -> str: - return f"""show views in database "{db_name}";""" + # SHOW VIEWS can return a maximum of 10000 rows. + # https://docs.snowflake.com/en/sql-reference/sql/show-views#usage-notes + assert limit <= SHOW_VIEWS_MAX_PAGE_SIZE - @staticmethod - def show_views_for_schema(schema_name: str, db_name: Optional[str]) -> str: - db_clause = f'"{db_name}".' if db_name is not None else "" - return f"""show views in schema {db_clause}"{schema_name}";""" + # To work around this, we paginate through the results using the FROM clause. + from_clause = ( + f"""FROM '{view_pagination_marker}'""" if view_pagination_marker else "" + ) + return f"""\ +SHOW VIEWS IN DATABASE "{db_name}" +LIMIT {limit} {from_clause}; +""" @staticmethod - def columns_for_schema(schema_name: str, db_name: Optional[str]) -> str: - db_clause = f'"{db_name}".' if db_name is not None else "" - return f""" - select - table_catalog AS "TABLE_CATALOG", - table_schema AS "TABLE_SCHEMA", - table_name AS "TABLE_NAME", - column_name AS "COLUMN_NAME", - ordinal_position AS "ORDINAL_POSITION", - is_nullable AS "IS_NULLABLE", - data_type AS "DATA_TYPE", - comment AS "COMMENT", - character_maximum_length AS "CHARACTER_MAXIMUM_LENGTH", - numeric_precision AS "NUMERIC_PRECISION", - numeric_scale AS "NUMERIC_SCALE", - column_default AS "COLUMN_DEFAULT", - is_identity AS "IS_IDENTITY" - from {db_clause}information_schema.columns - WHERE table_schema='{schema_name}' - ORDER BY ordinal_position""" - - @staticmethod - def columns_for_table( - table_name: str, schema_name: str, db_name: Optional[str] + def columns_for_schema( + schema_name: str, + db_name: str, + prefix_groups: Optional[List[PrefixGroup]] = None, ) -> str: - db_clause = f'"{db_name}".' if db_name is not None else "" - return f""" - select - table_catalog AS "TABLE_CATALOG", - table_schema AS "TABLE_SCHEMA", - table_name AS "TABLE_NAME", - column_name AS "COLUMN_NAME", - ordinal_position AS "ORDINAL_POSITION", - is_nullable AS "IS_NULLABLE", - data_type AS "DATA_TYPE", - comment AS "COMMENT", - character_maximum_length AS "CHARACTER_MAXIMUM_LENGTH", - numeric_precision AS "NUMERIC_PRECISION", - numeric_scale AS "NUMERIC_SCALE", - column_default AS "COLUMN_DEFAULT", - is_identity AS "IS_IDENTITY" - from {db_clause}information_schema.columns - WHERE table_schema='{schema_name}' and table_name='{table_name}' - ORDER BY ordinal_position""" + columns_template = """\ +SELECT + table_catalog AS "TABLE_CATALOG", + table_schema AS "TABLE_SCHEMA", + table_name AS "TABLE_NAME", + column_name AS "COLUMN_NAME", + ordinal_position AS "ORDINAL_POSITION", + is_nullable AS "IS_NULLABLE", + data_type AS "DATA_TYPE", + comment AS "COMMENT", + character_maximum_length AS "CHARACTER_MAXIMUM_LENGTH", + numeric_precision AS "NUMERIC_PRECISION", + numeric_scale AS "NUMERIC_SCALE", + column_default AS "COLUMN_DEFAULT", + is_identity AS "IS_IDENTITY" +FROM "{db_name}".information_schema.columns +WHERE table_schema='{schema_name}' AND {extra_clause}""" + + selects = [] + if prefix_groups is None: + prefix_groups = [PrefixGroup(prefix="", names=[])] + for prefix_group in prefix_groups: + if prefix_group.prefix == "": + extra_clause = "TRUE" + elif prefix_group.exact_match: + extra_clause = f"table_name = '{prefix_group.prefix}'" + else: + extra_clause = f"table_name LIKE '{prefix_group.prefix}%'" + + selects.append( + columns_template.format( + db_name=db_name, schema_name=schema_name, extra_clause=extra_clause + ) + ) + + return ( + "\nUNION ALL\n".join(selects) + + """\nORDER BY table_name, ordinal_position""" + ) @staticmethod def show_primary_keys_for_schema(schema_name: str, db_name: str) -> str: @@ -344,45 +331,6 @@ def operational_data_for_time_window( ORDER BY query_start_time DESC ;""" - @staticmethod - def table_to_table_lineage_history( - start_time_millis: int, - end_time_millis: int, - include_column_lineage: bool = True, - ) -> str: - return f""" - WITH table_lineage_history AS ( - SELECT - r.value:"objectName"::varchar AS upstream_table_name, - r.value:"objectDomain"::varchar AS upstream_table_domain, - r.value:"columns" AS upstream_table_columns, - w.value:"objectName"::varchar AS downstream_table_name, - w.value:"objectDomain"::varchar AS downstream_table_domain, - w.value:"columns" AS downstream_table_columns, - t.query_start_time AS query_start_time - FROM - (SELECT * from snowflake.account_usage.access_history) t, - lateral flatten(input => t.DIRECT_OBJECTS_ACCESSED) r, - lateral flatten(input => t.OBJECTS_MODIFIED) w - WHERE r.value:"objectId" IS NOT NULL - AND w.value:"objectId" IS NOT NULL - AND w.value:"objectName" NOT LIKE '%.GE_TMP_%' - AND w.value:"objectName" NOT LIKE '%.GE_TEMP_%' - AND t.query_start_time >= to_timestamp_ltz({start_time_millis}, 3) - AND t.query_start_time < to_timestamp_ltz({end_time_millis}, 3)) - SELECT - upstream_table_name AS "UPSTREAM_TABLE_NAME", - downstream_table_name AS "DOWNSTREAM_TABLE_NAME", - upstream_table_columns AS "UPSTREAM_TABLE_COLUMNS", - downstream_table_columns AS "DOWNSTREAM_TABLE_COLUMNS" - FROM table_lineage_history - WHERE upstream_table_domain in ('Table', 'External table') and downstream_table_domain = 'Table' - QUALIFY ROW_NUMBER() OVER ( - PARTITION BY downstream_table_name, - upstream_table_name{", downstream_table_columns" if include_column_lineage else ""} - ORDER BY query_start_time DESC - ) = 1""" - @staticmethod def view_dependencies() -> str: return """ @@ -403,58 +351,6 @@ def view_dependencies() -> str: referencing_object_domain in ('VIEW', 'MATERIALIZED VIEW') """ - @staticmethod - def view_lineage_history( - start_time_millis: int, - end_time_millis: int, - include_column_lineage: bool = True, - ) -> str: - return f""" - WITH view_lineage_history AS ( - SELECT - vu.value : "objectName"::varchar AS view_name, - vu.value : "objectDomain"::varchar AS view_domain, - vu.value : "columns" AS view_columns, - w.value : "objectName"::varchar AS downstream_table_name, - w.value : "objectDomain"::varchar AS downstream_table_domain, - w.value : "columns" AS downstream_table_columns, - t.query_start_time AS query_start_time - FROM - ( - SELECT - * - FROM - snowflake.account_usage.access_history - ) t, - lateral flatten(input => t.DIRECT_OBJECTS_ACCESSED) vu, - lateral flatten(input => t.OBJECTS_MODIFIED) w - WHERE - vu.value : "objectId" IS NOT NULL - AND w.value : "objectId" IS NOT NULL - AND w.value : "objectName" NOT LIKE '%.GE_TMP_%' - AND w.value : "objectName" NOT LIKE '%.GE_TEMP_%' - AND t.query_start_time >= to_timestamp_ltz({start_time_millis}, 3) - AND t.query_start_time < to_timestamp_ltz({end_time_millis}, 3) - ) - SELECT - view_name AS "VIEW_NAME", - view_domain AS "VIEW_DOMAIN", - view_columns AS "VIEW_COLUMNS", - downstream_table_name AS "DOWNSTREAM_TABLE_NAME", - downstream_table_domain AS "DOWNSTREAM_TABLE_DOMAIN", - downstream_table_columns AS "DOWNSTREAM_TABLE_COLUMNS" - FROM - view_lineage_history - WHERE - view_domain in ('View', 'Materialized view') - QUALIFY ROW_NUMBER() OVER ( - PARTITION BY view_name, - downstream_table_name {", downstream_table_columns" if include_column_lineage else ""} - ORDER BY - query_start_time DESC - ) = 1 - """ - # Note on use of `upstreams_deny_pattern` to ignore temporary tables: # Snowflake access history may include temporary tables in DIRECT_OBJECTS_ACCESSED and # OBJECTS_MODIFIED->columns->directSources. We do not need these temporary tables and filter these in the query. @@ -464,7 +360,7 @@ def table_to_table_lineage_history_v2( end_time_millis: int, include_view_lineage: bool = True, include_column_lineage: bool = True, - upstreams_deny_pattern: List[str] = DEFAULT_TABLES_DENY_LIST, + upstreams_deny_pattern: List[str] = DEFAULT_TEMP_TABLES_PATTERNS, ) -> str: if include_column_lineage: return SnowflakeQuery.table_upstreams_with_column_lineage( @@ -515,7 +411,7 @@ def show_external_tables() -> str: def copy_lineage_history( start_time_millis: int, end_time_millis: int, - downstreams_deny_pattern: List[str] = DEFAULT_TABLES_DENY_LIST, + downstreams_deny_pattern: List[str] = DEFAULT_TEMP_TABLES_PATTERNS, ) -> str: temp_table_filter = create_deny_regex_sql_filter( downstreams_deny_pattern, @@ -558,7 +454,7 @@ def usage_per_object_per_time_bucket_for_time_window( include_top_n_queries: bool, email_domain: Optional[str], email_filter: AllowDenyPattern, - table_deny_pattern: List[str] = DEFAULT_TABLES_DENY_LIST, + table_deny_pattern: List[str] = DEFAULT_TEMP_TABLES_PATTERNS, ) -> str: if not include_top_n_queries: top_n_queries = 0 @@ -775,6 +671,9 @@ def table_upstreams_with_column_lineage( upstreams_deny_pattern, ["upstream_table_name", "upstream_column_table_name"], ) + _MAX_UPSTREAMS_PER_DOWNSTREAM = 20 + _MAX_UPSTREAM_COLUMNS_PER_DOWNSTREAM = 400 + _MAX_QUERIES_PER_DOWNSTREAM = 30 return f""" WITH column_lineage_history AS ( @@ -790,7 +689,7 @@ def table_upstreams_with_column_lineage( t.query_start_time AS query_start_time, t.query_id AS query_id FROM - (SELECT * from snowflake.account_usage.access_history) t, + snowflake.account_usage.access_history t, lateral flatten(input => t.DIRECT_OBJECTS_ACCESSED) r, lateral flatten(input => t.OBJECTS_MODIFIED) w, lateral flatten(input => w.value : "columns", outer => true) wcols, @@ -886,30 +785,32 @@ def table_upstreams_with_column_lineage( AND query_history.start_time < to_timestamp_ltz({end_time_millis}, 3) ) query_history on qid.query_id = query_history.query_id + WHERE qid.query_id is not null + AND query_history.query_text is not null ) SELECT h.downstream_table_name AS "DOWNSTREAM_TABLE_NAME", ANY_VALUE(h.downstream_table_domain) AS "DOWNSTREAM_TABLE_DOMAIN", - ARRAY_UNIQUE_AGG( + ARRAY_SLICE(ARRAY_UNIQUE_AGG( OBJECT_CONSTRUCT( 'upstream_object_name', h.upstream_table_name, 'upstream_object_domain', h.upstream_table_domain, 'query_id', h.query_id ) - ) AS "UPSTREAM_TABLES", - ARRAY_UNIQUE_AGG( + ), 0, {_MAX_UPSTREAMS_PER_DOWNSTREAM}) AS "UPSTREAM_TABLES", + ARRAY_SLICE(ARRAY_UNIQUE_AGG( OBJECT_CONSTRUCT( - 'column_name', column_upstreams.downstream_column_name, - 'upstreams', column_upstreams.upstreams + 'column_name', column_upstreams.downstream_column_name, + 'upstreams', column_upstreams.upstreams ) - ) AS "UPSTREAM_COLUMNS", - ARRAY_UNIQUE_AGG( + ), 0, {_MAX_UPSTREAM_COLUMNS_PER_DOWNSTREAM}) AS "UPSTREAM_COLUMNS", + ARRAY_SLICE(ARRAY_UNIQUE_AGG( OBJECT_CONSTRUCT( 'query_id', q.query_id, 'query_text', q.query_text, 'start_time', q.start_time ) - ) as "QUERIES" + ), 0, {_MAX_QUERIES_PER_DOWNSTREAM}) as "QUERIES" FROM table_upstream_jobs_unique h LEFT JOIN column_upstreams column_upstreams @@ -950,7 +851,7 @@ def table_upstreams_only( t.query_start_time AS query_start_time, t.query_id AS query_id FROM - (SELECT * from snowflake.account_usage.access_history) t, + snowflake.account_usage.access_history t, lateral flatten(input => t.DIRECT_OBJECTS_ACCESSED) r, lateral flatten(input => t.OBJECTS_MODIFIED) w WHERE @@ -1016,3 +917,25 @@ def table_upstreams_only( ORDER BY h.downstream_table_name """ + + @staticmethod + def dmf_assertion_results(start_time_millis: int, end_time_millis: int) -> str: + pattern = r"datahub\\_\\_%" + escape_pattern = r"\\" + return f""" + SELECT + MEASUREMENT_TIME AS "MEASUREMENT_TIME", + METRIC_NAME AS "METRIC_NAME", + TABLE_NAME AS "TABLE_NAME", + TABLE_SCHEMA AS "TABLE_SCHEMA", + TABLE_DATABASE AS "TABLE_DATABASE", + VALUE::INT AS "VALUE" + FROM + SNOWFLAKE.LOCAL.DATA_QUALITY_MONITORING_RESULTS + WHERE + MEASUREMENT_TIME >= to_timestamp_ltz({start_time_millis}, 3) + AND MEASUREMENT_TIME < to_timestamp_ltz({end_time_millis}, 3) + AND METRIC_NAME ilike '{pattern}' escape '{escape_pattern}' + ORDER BY MEASUREMENT_TIME ASC; + +""" diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py index d79ed384d755b..4924546383aa4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field from datetime import datetime -from typing import Dict, List, MutableSet, Optional +from typing import TYPE_CHECKING, Dict, List, MutableSet, Optional from datahub.ingestion.api.report import Report from datahub.ingestion.glossary.classification_mixin import ClassificationReportMixin @@ -14,6 +14,11 @@ from datahub.sql_parsing.sql_parsing_aggregator import SqlAggregatorReport from datahub.utilities.perf_timer import PerfTimer +if TYPE_CHECKING: + from datahub.ingestion.source.snowflake.snowflake_schema import ( + SnowflakeDataDictionary, + ) + @dataclass class SnowflakeUsageAggregationReport(Report): @@ -106,18 +111,12 @@ class SnowflakeV2Report( num_tables_with_known_upstreams: int = 0 num_upstream_lineage_edge_parsing_failed: int = 0 - # Reports how many times we reset in-memory `functools.lru_cache` caches of data, - # which occurs when we occur a different database / schema. - # Should not be more than the number of databases / schemas scanned. - # Maps (function name) -> (stat_name) -> (stat_value) - lru_cache_info: Dict[str, Dict[str, int]] = field(default_factory=dict) + data_dictionary_cache: Optional["SnowflakeDataDictionary"] = None # These will be non-zero if snowflake information_schema queries fail with error - # "Information schema query returned too much data. Please repeat query with more selective predicates."" # This will result in overall increase in time complexity num_get_tables_for_schema_queries: int = 0 - num_get_views_for_schema_queries: int = 0 - num_get_columns_for_table_queries: int = 0 # these will be non-zero if the user choses to enable the extract_tags = "with_lineage" option, which requires # individual queries per object (database, schema, table) and an extra query per table to get the tags on the columns. diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py index 292c57494632c..ce8f20d23aa6b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py @@ -1,19 +1,26 @@ import logging +import os from collections import defaultdict from dataclasses import dataclass, field from datetime import datetime -from functools import lru_cache -from typing import Dict, List, Optional - -from snowflake.connector import SnowflakeConnection +from typing import Callable, Dict, Iterable, List, MutableMapping, Optional +from datahub.ingestion.api.report import SupportsAsObj from datahub.ingestion.source.snowflake.constants import SnowflakeObjectDomain -from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery -from datahub.ingestion.source.snowflake.snowflake_utils import SnowflakeQueryMixin +from datahub.ingestion.source.snowflake.snowflake_connection import SnowflakeConnection +from datahub.ingestion.source.snowflake.snowflake_query import ( + SHOW_VIEWS_MAX_PAGE_SIZE, + SnowflakeQuery, +) from datahub.ingestion.source.sql.sql_generic import BaseColumn, BaseTable, BaseView +from datahub.utilities.file_backed_collections import FileBackedDict +from datahub.utilities.prefix_batch_builder import build_prefix_batches +from datahub.utilities.serialized_lru_cache import serialized_lru_cache logger: logging.Logger = logging.getLogger(__name__) +SCHEMA_PARALLELISM = int(os.getenv("DATAHUB_SNOWFLAKE_SCHEMA_PARALLELISM", 20)) + @dataclass class SnowflakePK: @@ -176,23 +183,36 @@ def get_column_tags_for_table( ) -class SnowflakeDataDictionary(SnowflakeQueryMixin): - def __init__(self) -> None: +class SnowflakeDataDictionary(SupportsAsObj): + def __init__(self, connection: SnowflakeConnection) -> None: self.logger = logger - self.connection: Optional[SnowflakeConnection] = None - def set_connection(self, connection: SnowflakeConnection) -> None: self.connection = connection - def get_connection(self) -> SnowflakeConnection: - # Connection is already present by the time this is called - assert self.connection is not None - return self.connection + def as_obj(self) -> Dict[str, Dict[str, int]]: + # TODO: Move this into a proper report type that gets computed. + + # Reports how many times we reset in-memory `functools.lru_cache` caches of data, + # which occurs when we occur a different database / schema. + # Should not be more than the number of databases / schemas scanned. + # Maps (function name) -> (stat_name) -> (stat_value) + lru_cache_functions: List[Callable] = [ + self.get_tables_for_database, + self.get_views_for_database, + self.get_columns_for_schema, + self.get_pk_constraints_for_schema, + self.get_fk_constraints_for_schema, + ] + + report = {} + for func in lru_cache_functions: + report[func.__name__] = func.cache_info()._asdict() # type: ignore + return report def show_databases(self) -> List[SnowflakeDatabase]: databases: List[SnowflakeDatabase] = [] - cur = self.query( + cur = self.connection.query( SnowflakeQuery.show_databases(), ) @@ -209,7 +229,7 @@ def show_databases(self) -> List[SnowflakeDatabase]: def get_databases(self, db_name: str) -> List[SnowflakeDatabase]: databases: List[SnowflakeDatabase] = [] - cur = self.query( + cur = self.connection.query( SnowflakeQuery.get_databases(db_name), ) @@ -227,7 +247,7 @@ def get_databases(self, db_name: str) -> List[SnowflakeDatabase]: def get_schemas_for_database(self, db_name: str) -> List[SnowflakeSchema]: snowflake_schemas = [] - cur = self.query( + cur = self.connection.query( SnowflakeQuery.schemas_for_database(db_name), ) @@ -241,13 +261,13 @@ def get_schemas_for_database(self, db_name: str) -> List[SnowflakeSchema]: snowflake_schemas.append(snowflake_schema) return snowflake_schemas - @lru_cache(maxsize=1) + @serialized_lru_cache(maxsize=1) def get_tables_for_database( self, db_name: str ) -> Optional[Dict[str, List[SnowflakeTable]]]: tables: Dict[str, List[SnowflakeTable]] = {} try: - cur = self.query( + cur = self.connection.query( SnowflakeQuery.tables_for_database(db_name), ) except Exception as e: @@ -280,7 +300,7 @@ def get_tables_for_schema( ) -> List[SnowflakeTable]: tables: List[SnowflakeTable] = [] - cur = self.query( + cur = self.connection.query( SnowflakeQuery.tables_for_schema(schema_name, db_name), ) @@ -299,118 +319,109 @@ def get_tables_for_schema( ) return tables - @lru_cache(maxsize=1) - def get_views_for_database( - self, db_name: str - ) -> Optional[Dict[str, List[SnowflakeView]]]: + @serialized_lru_cache(maxsize=1) + def get_views_for_database(self, db_name: str) -> Dict[str, List[SnowflakeView]]: + page_limit = SHOW_VIEWS_MAX_PAGE_SIZE + views: Dict[str, List[SnowflakeView]] = {} - try: - cur = self.query(SnowflakeQuery.show_views_for_database(db_name)) - except Exception as e: - logger.debug( - f"Failed to get all views for database - {db_name}", exc_info=e - ) - # Error - Information schema query returned too much data. Please repeat query with more selective predicates. - return None - for table in cur: - if table["schema_name"] not in views: - views[table["schema_name"]] = [] - views[table["schema_name"]].append( - SnowflakeView( - name=table["name"], - created=table["created_on"], - # last_altered=table["last_altered"], - comment=table["comment"], - view_definition=table["text"], - last_altered=table["created_on"], - materialized=table.get("is_materialized", "false").lower() - == "true", + first_iteration = True + view_pagination_marker: Optional[str] = None + while first_iteration or view_pagination_marker is not None: + cur = self.connection.query( + SnowflakeQuery.show_views_for_database( + db_name, + limit=page_limit, + view_pagination_marker=view_pagination_marker, ) ) - return views - def get_views_for_schema( - self, schema_name: str, db_name: str - ) -> List[SnowflakeView]: - views: List[SnowflakeView] = [] + first_iteration = False + view_pagination_marker = None + + result_set_size = 0 + for view in cur: + result_set_size += 1 + + view_name = view["name"] + schema_name = view["schema_name"] + if schema_name not in views: + views[schema_name] = [] + views[schema_name].append( + SnowflakeView( + name=view_name, + created=view["created_on"], + # last_altered=table["last_altered"], + comment=view["comment"], + view_definition=view["text"], + last_altered=view["created_on"], + materialized=( + view.get("is_materialized", "false").lower() == "true" + ), + ) + ) - cur = self.query(SnowflakeQuery.show_views_for_schema(schema_name, db_name)) - for table in cur: - views.append( - SnowflakeView( - name=table["name"], - created=table["created_on"], - # last_altered=table["last_altered"], - comment=table["comment"], - view_definition=table["text"], - last_altered=table["created_on"], + if result_set_size >= page_limit: + # If we hit the limit, we need to send another request to get the next page. + logger.info( + f"Fetching next page of views for {db_name} - after {view_name}" ) - ) + view_pagination_marker = view_name + return views - @lru_cache(maxsize=1) + @serialized_lru_cache(maxsize=SCHEMA_PARALLELISM) def get_columns_for_schema( - self, schema_name: str, db_name: str - ) -> Optional[Dict[str, List[SnowflakeColumn]]]: - columns: Dict[str, List[SnowflakeColumn]] = {} - try: - cur = self.query(SnowflakeQuery.columns_for_schema(schema_name, db_name)) - except Exception as e: - logger.debug( - f"Failed to get all columns for schema - {schema_name}", exc_info=e - ) - # Error - Information schema query returned too much data. - # Please repeat query with more selective predicates. - return None - - for column in cur: - if column["TABLE_NAME"] not in columns: - columns[column["TABLE_NAME"]] = [] - columns[column["TABLE_NAME"]].append( - SnowflakeColumn( - name=column["COLUMN_NAME"], - ordinal_position=column["ORDINAL_POSITION"], - is_nullable=column["IS_NULLABLE"] == "YES", - data_type=column["DATA_TYPE"], - comment=column["COMMENT"], - character_maximum_length=column["CHARACTER_MAXIMUM_LENGTH"], - numeric_precision=column["NUMERIC_PRECISION"], - numeric_scale=column["NUMERIC_SCALE"], + self, + schema_name: str, + db_name: str, + # HACK: This key is excluded from the cache key. + cache_exclude_all_objects: Iterable[str], + ) -> MutableMapping[str, List[SnowflakeColumn]]: + all_objects = list(cache_exclude_all_objects) + + columns: MutableMapping[str, List[SnowflakeColumn]] = {} + if len(all_objects) > 10000: + # For massive schemas, use a FileBackedDict to avoid memory issues. + columns = FileBackedDict() + + object_batches = build_prefix_batches( + all_objects, max_batch_size=10000, max_groups_in_batch=5 + ) + for batch_index, object_batch in enumerate(object_batches): + if batch_index > 0: + logger.info( + f"Still fetching columns for {db_name}.{schema_name} - batch {batch_index + 1} of {len(object_batches)}" ) + query = SnowflakeQuery.columns_for_schema( + schema_name, db_name, object_batch ) - return columns - - def get_columns_for_table( - self, table_name: str, schema_name: str, db_name: str - ) -> List[SnowflakeColumn]: - columns: List[SnowflakeColumn] = [] - - cur = self.query( - SnowflakeQuery.columns_for_table(table_name, schema_name, db_name), - ) - for column in cur: - columns.append( - SnowflakeColumn( - name=column["COLUMN_NAME"], - ordinal_position=column["ORDINAL_POSITION"], - is_nullable=column["IS_NULLABLE"] == "YES", - data_type=column["DATA_TYPE"], - comment=column["COMMENT"], - character_maximum_length=column["CHARACTER_MAXIMUM_LENGTH"], - numeric_precision=column["NUMERIC_PRECISION"], - numeric_scale=column["NUMERIC_SCALE"], + cur = self.connection.query(query) + + for column in cur: + if column["TABLE_NAME"] not in columns: + columns[column["TABLE_NAME"]] = [] + columns[column["TABLE_NAME"]].append( + SnowflakeColumn( + name=column["COLUMN_NAME"], + ordinal_position=column["ORDINAL_POSITION"], + is_nullable=column["IS_NULLABLE"] == "YES", + data_type=column["DATA_TYPE"], + comment=column["COMMENT"], + character_maximum_length=column["CHARACTER_MAXIMUM_LENGTH"], + numeric_precision=column["NUMERIC_PRECISION"], + numeric_scale=column["NUMERIC_SCALE"], + ) ) - ) return columns - @lru_cache(maxsize=1) + @serialized_lru_cache(maxsize=SCHEMA_PARALLELISM) def get_pk_constraints_for_schema( self, schema_name: str, db_name: str ) -> Dict[str, SnowflakePK]: constraints: Dict[str, SnowflakePK] = {} - cur = self.query( + cur = self.connection.query( SnowflakeQuery.show_primary_keys_for_schema(schema_name, db_name), ) @@ -422,14 +433,14 @@ def get_pk_constraints_for_schema( constraints[row["table_name"]].column_names.append(row["column_name"]) return constraints - @lru_cache(maxsize=1) + @serialized_lru_cache(maxsize=SCHEMA_PARALLELISM) def get_fk_constraints_for_schema( self, schema_name: str, db_name: str ) -> Dict[str, List[SnowflakeFK]]: constraints: Dict[str, List[SnowflakeFK]] = {} fk_constraints_map: Dict[str, SnowflakeFK] = {} - cur = self.query( + cur = self.connection.query( SnowflakeQuery.show_foreign_keys_for_schema(schema_name, db_name), ) @@ -461,7 +472,7 @@ def get_tags_for_database_without_propagation( self, db_name: str, ) -> _SnowflakeTagCache: - cur = self.query( + cur = self.connection.query( SnowflakeQuery.get_all_tags_in_database_without_propagation(db_name) ) @@ -516,7 +527,7 @@ def get_tags_for_object_with_propagation( ) -> List[SnowflakeTag]: tags: List[SnowflakeTag] = [] - cur = self.query( + cur = self.connection.query( SnowflakeQuery.get_all_tags_on_object_with_propagation( db_name, quoted_identifier, domain ), @@ -537,7 +548,7 @@ def get_tags_on_columns_for_table( self, quoted_table_name: str, db_name: str ) -> Dict[str, List[SnowflakeTag]]: tags: Dict[str, List[SnowflakeTag]] = defaultdict(list) - cur = self.query( + cur = self.connection.query( SnowflakeQuery.get_tags_on_columns_with_propagation( db_name, quoted_table_name ), diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py new file mode 100644 index 0000000000000..e604ed96b8eb6 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -0,0 +1,1058 @@ +import concurrent.futures +import itertools +import logging +import queue +from typing import Callable, Dict, Iterable, List, Optional, Union + +from datahub.configuration.pattern_utils import is_schema_allowed +from datahub.emitter.mce_builder import ( + make_data_platform_urn, + make_dataset_urn_with_platform_instance, + make_schema_field_urn, + make_tag_urn, +) +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.api.source import SourceReport +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.glossary.classification_mixin import ( + ClassificationHandler, + classification_workunit_processor, +) +from datahub.ingestion.source.common.subtypes import ( + DatasetContainerSubTypes, + DatasetSubTypes, +) +from datahub.ingestion.source.snowflake.constants import ( + GENERIC_PERMISSION_ERROR_KEY, + SNOWFLAKE_DATABASE, + SnowflakeObjectDomain, +) +from datahub.ingestion.source.snowflake.snowflake_config import ( + SnowflakeFilterConfig, + SnowflakeIdentifierConfig, + SnowflakeV2Config, + TagOption, +) +from datahub.ingestion.source.snowflake.snowflake_connection import ( + SnowflakeConnection, + SnowflakePermissionError, +) +from datahub.ingestion.source.snowflake.snowflake_data_reader import SnowflakeDataReader +from datahub.ingestion.source.snowflake.snowflake_profiler import SnowflakeProfiler +from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report +from datahub.ingestion.source.snowflake.snowflake_schema import ( + SCHEMA_PARALLELISM, + SnowflakeColumn, + SnowflakeDatabase, + SnowflakeDataDictionary, + SnowflakeFK, + SnowflakePK, + SnowflakeSchema, + SnowflakeTable, + SnowflakeTag, + SnowflakeView, +) +from datahub.ingestion.source.snowflake.snowflake_tag import SnowflakeTagExtractor +from datahub.ingestion.source.snowflake.snowflake_utils import ( + SnowflakeFilterMixin, + SnowflakeIdentifierMixin, + SnowsightUrlBuilder, +) +from datahub.ingestion.source.sql.sql_utils import ( + add_table_to_schema_container, + gen_database_container, + gen_database_key, + gen_schema_container, + gen_schema_key, + get_dataplatform_instance_aspect, + get_domain_wu, +) +from datahub.ingestion.source_report.ingestion_stage import ( + METADATA_EXTRACTION, + PROFILING, +) +from datahub.metadata.com.linkedin.pegasus2avro.common import ( + GlobalTags, + Status, + SubTypes, + TagAssociation, + TimeStamp, +) +from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( + DatasetProperties, + ViewProperties, +) +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + ArrayType, + BooleanType, + BytesType, + DateType, + ForeignKeyConstraint, + MySqlDDL, + NullType, + NumberType, + RecordType, + SchemaField, + SchemaFieldDataType, + SchemaMetadata, + StringType, + TimeType, +) +from datahub.metadata.com.linkedin.pegasus2avro.tag import TagProperties +from datahub.sql_parsing.sql_parsing_aggregator import SqlParsingAggregator +from datahub.utilities.registries.domain_registry import DomainRegistry + +logger = logging.getLogger(__name__) + +# https://docs.snowflake.com/en/sql-reference/intro-summary-data-types.html +SNOWFLAKE_FIELD_TYPE_MAPPINGS = { + "DATE": DateType, + "BIGINT": NumberType, + "BINARY": BytesType, + # 'BIT': BIT, + "BOOLEAN": BooleanType, + "CHAR": NullType, + "CHARACTER": NullType, + "DATETIME": TimeType, + "DEC": NumberType, + "DECIMAL": NumberType, + "DOUBLE": NumberType, + "FIXED": NumberType, + "FLOAT": NumberType, + "INT": NumberType, + "INTEGER": NumberType, + "NUMBER": NumberType, + # 'OBJECT': ? + "REAL": NumberType, + "BYTEINT": NumberType, + "SMALLINT": NumberType, + "STRING": StringType, + "TEXT": StringType, + "TIME": TimeType, + "TIMESTAMP": TimeType, + "TIMESTAMP_TZ": TimeType, + "TIMESTAMP_LTZ": TimeType, + "TIMESTAMP_NTZ": TimeType, + "TINYINT": NumberType, + "VARBINARY": BytesType, + "VARCHAR": StringType, + "VARIANT": RecordType, + "OBJECT": NullType, + "ARRAY": ArrayType, + "GEOGRAPHY": NullType, +} + + +class SnowflakeSchemaGenerator(SnowflakeFilterMixin, SnowflakeIdentifierMixin): + def __init__( + self, + config: SnowflakeV2Config, + report: SnowflakeV2Report, + connection: SnowflakeConnection, + dataset_urn_builder: Callable[[str], str], + domain_registry: Optional[DomainRegistry], + profiler: Optional[SnowflakeProfiler], + aggregator: Optional[SqlParsingAggregator], + snowsight_url_builder: Optional[SnowsightUrlBuilder], + ) -> None: + self.config: SnowflakeV2Config = config + self.report: SnowflakeV2Report = report + self.connection: SnowflakeConnection = connection + self.dataset_urn_builder = dataset_urn_builder + + self.data_dictionary: SnowflakeDataDictionary = SnowflakeDataDictionary( + connection=self.connection + ) + self.report.data_dictionary_cache = self.data_dictionary + + self.domain_registry: Optional[DomainRegistry] = domain_registry + self.classification_handler = ClassificationHandler(self.config, self.report) + self.tag_extractor = SnowflakeTagExtractor( + config, self.data_dictionary, self.report + ) + self.profiler: Optional[SnowflakeProfiler] = profiler + self.snowsight_url_builder: Optional[ + SnowsightUrlBuilder + ] = snowsight_url_builder + + # These are populated as side-effects of get_workunits_internal. + self.databases: List[SnowflakeDatabase] = [] + self.aggregator: Optional[SqlParsingAggregator] = aggregator + + def get_connection(self) -> SnowflakeConnection: + return self.connection + + @property + def structured_reporter(self) -> SourceReport: + return self.report + + @property + def filter_config(self) -> SnowflakeFilterConfig: + return self.config + + @property + def identifier_config(self) -> SnowflakeIdentifierConfig: + return self.config + + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: + self.databases = [] + for database in self.get_databases() or []: + self.report.report_entity_scanned(database.name, "database") + if not self.filter_config.database_pattern.allowed(database.name): + self.report.report_dropped(f"{database.name}.*") + else: + self.databases.append(database) + + if len(self.databases) == 0: + return + + try: + for snowflake_db in self.databases: + self.report.set_ingestion_stage(snowflake_db.name, METADATA_EXTRACTION) + yield from self._process_database(snowflake_db) + + except SnowflakePermissionError as e: + self.report_error(GENERIC_PERMISSION_ERROR_KEY, str(e)) + return + + def get_databases(self) -> Optional[List[SnowflakeDatabase]]: + try: + # `show databases` is required only to get one of the databases + # whose information_schema can be queried to start with. + databases = self.data_dictionary.show_databases() + except Exception as e: + logger.debug(f"Failed to list databases due to error {e}", exc_info=e) + self.report_error( + "list-databases", + f"Failed to list databases due to error {e}", + ) + return None + else: + ischema_databases: List[ + SnowflakeDatabase + ] = self.get_databases_from_ischema(databases) + + if len(ischema_databases) == 0: + self.report_error( + GENERIC_PERMISSION_ERROR_KEY, + "No databases found. Please check permissions.", + ) + return ischema_databases + + def get_databases_from_ischema( + self, databases: List[SnowflakeDatabase] + ) -> List[SnowflakeDatabase]: + ischema_databases: List[SnowflakeDatabase] = [] + for database in databases: + try: + ischema_databases = self.data_dictionary.get_databases(database.name) + break + except Exception: + # query fails if "USAGE" access is not granted for database + # This is okay, because `show databases` query lists all databases irrespective of permission, + # if role has `MANAGE GRANTS` privilege. (not advisable) + logger.debug( + f"Failed to list databases {database.name} information_schema" + ) + # SNOWFLAKE database always shows up even if permissions are missing + if database == SNOWFLAKE_DATABASE: + continue + logger.info( + f"The role {self.report.role} has `MANAGE GRANTS` privilege. This is not advisable and also not required." + ) + + return ischema_databases + + def _process_database( + self, snowflake_db: SnowflakeDatabase + ) -> Iterable[MetadataWorkUnit]: + db_name = snowflake_db.name + + try: + pass + # self.query(SnowflakeQuery.use_database(db_name)) + except Exception as e: + if isinstance(e, SnowflakePermissionError): + # This may happen if REFERENCE_USAGE permissions are set + # We can not run show queries on database in such case. + # This need not be a failure case. + self.report_warning( + "Insufficient privileges to operate on database, skipping. Please grant USAGE permissions on database to extract its metadata.", + db_name, + ) + else: + logger.debug( + f"Failed to use database {db_name} due to error {e}", + exc_info=e, + ) + self.report_warning( + "Failed to get schemas for database", + db_name, + ) + return + + if self.config.extract_tags != TagOption.skip: + snowflake_db.tags = self.tag_extractor.get_tags_on_object( + domain="database", db_name=db_name + ) + + if self.config.include_technical_schema: + yield from self.gen_database_containers(snowflake_db) + + self.fetch_schemas_for_database(snowflake_db, db_name) + + if self.config.include_technical_schema and snowflake_db.tags: + for tag in snowflake_db.tags: + yield from self._process_tag(tag) + + # Caches tables for a single database. Consider moving to disk or S3 when possible. + db_tables: Dict[str, List[SnowflakeTable]] = {} + yield from self._process_db_schemas(snowflake_db, db_tables) + + if self.profiler and db_tables: + self.report.set_ingestion_stage(snowflake_db.name, PROFILING) + yield from self.profiler.get_workunits(snowflake_db, db_tables) + + def _process_db_schemas( + self, + snowflake_db: SnowflakeDatabase, + db_tables: Dict[str, List[SnowflakeTable]], + ) -> Iterable[MetadataWorkUnit]: + q: "queue.Queue[MetadataWorkUnit]" = queue.Queue(maxsize=100) + + def _process_schema_worker(snowflake_schema: SnowflakeSchema) -> None: + for wu in self._process_schema( + snowflake_schema, snowflake_db.name, db_tables + ): + q.put(wu) + + with concurrent.futures.ThreadPoolExecutor( + max_workers=SCHEMA_PARALLELISM + ) as executor: + futures = [] + for snowflake_schema in snowflake_db.schemas: + f = executor.submit(_process_schema_worker, snowflake_schema) + futures.append(f) + + # Read from the queue and yield the work units until all futures are done. + while True: + if not q.empty(): + while not q.empty(): + yield q.get_nowait() + else: + try: + yield q.get(timeout=0.2) + except queue.Empty: + pass + + # Filter out the done futures. + futures = [f for f in futures if not f.done()] + if not futures: + break + + # Yield the remaining work units. This theoretically should not happen, but adding it just in case. + while not q.empty(): + yield q.get_nowait() + + def fetch_schemas_for_database( + self, snowflake_db: SnowflakeDatabase, db_name: str + ) -> None: + schemas: List[SnowflakeSchema] = [] + try: + for schema in self.data_dictionary.get_schemas_for_database(db_name): + self.report.report_entity_scanned(schema.name, "schema") + if not is_schema_allowed( + self.filter_config.schema_pattern, + schema.name, + db_name, + self.filter_config.match_fully_qualified_names, + ): + self.report.report_dropped(f"{db_name}.{schema.name}.*") + else: + schemas.append(schema) + except Exception as e: + if isinstance(e, SnowflakePermissionError): + error_msg = f"Failed to get schemas for database {db_name}. Please check permissions." + # Ideal implementation would use PEP 678 – Enriching Exceptions with Notes + raise SnowflakePermissionError(error_msg) from e.__cause__ + else: + logger.debug( + f"Failed to get schemas for database {db_name} due to error {e}", + exc_info=e, + ) + self.report_warning( + "Failed to get schemas for database", + db_name, + ) + + if not schemas: + self.report_warning( + "No schemas found in database. If schemas exist, please grant USAGE permissions on them.", + db_name, + ) + else: + snowflake_db.schemas = schemas + + def _process_schema( + self, + snowflake_schema: SnowflakeSchema, + db_name: str, + db_tables: Dict[str, List[SnowflakeTable]], + ) -> Iterable[MetadataWorkUnit]: + schema_name = snowflake_schema.name + + if self.config.extract_tags != TagOption.skip: + snowflake_schema.tags = self.tag_extractor.get_tags_on_object( + schema_name=schema_name, db_name=db_name, domain="schema" + ) + + if self.config.include_technical_schema: + yield from self.gen_schema_containers(snowflake_schema, db_name) + + # We need to do this first so that we can use it when fetching columns. + if self.config.include_tables: + tables = self.fetch_tables_for_schema( + snowflake_schema, db_name, schema_name + ) + if self.config.include_views: + views = self.fetch_views_for_schema(snowflake_schema, db_name, schema_name) + + if self.config.include_tables: + db_tables[schema_name] = tables + + if self.config.include_technical_schema: + data_reader = self.make_data_reader() + for table in tables: + table_wu_generator = self._process_table( + table, snowflake_schema, db_name + ) + + yield from classification_workunit_processor( + table_wu_generator, + self.classification_handler, + data_reader, + [db_name, schema_name, table.name], + ) + + if self.config.include_views: + if ( + self.aggregator + and self.config.include_view_lineage + and self.config.parse_view_ddl + ): + for view in views: + view_identifier = self.get_dataset_identifier( + view.name, schema_name, db_name + ) + if view.view_definition: + self.aggregator.add_view_definition( + view_urn=self.dataset_urn_builder(view_identifier), + view_definition=view.view_definition, + default_db=db_name, + default_schema=schema_name, + ) + + if self.config.include_technical_schema: + for view in views: + yield from self._process_view(view, snowflake_schema, db_name) + + if self.config.include_technical_schema and snowflake_schema.tags: + for tag in snowflake_schema.tags: + yield from self._process_tag(tag) + + if not snowflake_schema.views and not snowflake_schema.tables: + self.report_warning( + "No tables/views found in schema. If tables exist, please grant REFERENCES or SELECT permissions on them.", + f"{db_name}.{schema_name}", + ) + + def fetch_views_for_schema( + self, snowflake_schema: SnowflakeSchema, db_name: str, schema_name: str + ) -> List[SnowflakeView]: + try: + views: List[SnowflakeView] = [] + for view in self.get_views_for_schema(schema_name, db_name): + view_name = self.get_dataset_identifier(view.name, schema_name, db_name) + + self.report.report_entity_scanned(view_name, "view") + + if not self.filter_config.view_pattern.allowed(view_name): + self.report.report_dropped(view_name) + else: + views.append(view) + snowflake_schema.views = [view.name for view in views] + return views + except Exception as e: + if isinstance(e, SnowflakePermissionError): + # Ideal implementation would use PEP 678 – Enriching Exceptions with Notes + error_msg = f"Failed to get views for schema {db_name}.{schema_name}. Please check permissions." + + raise SnowflakePermissionError(error_msg) from e.__cause__ + else: + logger.debug( + f"Failed to get views for schema {db_name}.{schema_name} due to error {e}", + exc_info=e, + ) + self.report_warning( + "Failed to get views for schema", + f"{db_name}.{schema_name}", + ) + return [] + + def fetch_tables_for_schema( + self, snowflake_schema: SnowflakeSchema, db_name: str, schema_name: str + ) -> List[SnowflakeTable]: + try: + tables: List[SnowflakeTable] = [] + for table in self.get_tables_for_schema(schema_name, db_name): + table_identifier = self.get_dataset_identifier( + table.name, schema_name, db_name + ) + self.report.report_entity_scanned(table_identifier) + if not self.filter_config.table_pattern.allowed(table_identifier): + self.report.report_dropped(table_identifier) + else: + tables.append(table) + snowflake_schema.tables = [table.name for table in tables] + return tables + except Exception as e: + if isinstance(e, SnowflakePermissionError): + # Ideal implementation would use PEP 678 – Enriching Exceptions with Notes + error_msg = f"Failed to get tables for schema {db_name}.{schema_name}. Please check permissions." + raise SnowflakePermissionError(error_msg) from e.__cause__ + else: + logger.debug( + f"Failed to get tables for schema {db_name}.{schema_name} due to error {e}", + exc_info=e, + ) + self.report_warning( + "Failed to get tables for schema", + f"{db_name}.{schema_name}", + ) + return [] + + def make_data_reader(self) -> Optional[SnowflakeDataReader]: + if self.classification_handler.is_classification_enabled() and self.connection: + return SnowflakeDataReader.create( + self.connection, self.snowflake_identifier + ) + + return None + + def _process_table( + self, + table: SnowflakeTable, + snowflake_schema: SnowflakeSchema, + db_name: str, + ) -> Iterable[MetadataWorkUnit]: + schema_name = snowflake_schema.name + table_identifier = self.get_dataset_identifier(table.name, schema_name, db_name) + + try: + table.columns = self.get_columns_for_table( + table.name, snowflake_schema, db_name + ) + table.column_count = len(table.columns) + if self.config.extract_tags != TagOption.skip: + table.column_tags = self.tag_extractor.get_column_tags_for_table( + table.name, schema_name, db_name + ) + except Exception as e: + logger.debug( + f"Failed to get columns for table {table_identifier} due to error {e}", + exc_info=e, + ) + self.report_warning("Failed to get columns for table", table_identifier) + + if self.config.extract_tags != TagOption.skip: + table.tags = self.tag_extractor.get_tags_on_object( + table_name=table.name, + schema_name=schema_name, + db_name=db_name, + domain="table", + ) + + if self.config.include_technical_schema: + if self.config.include_primary_keys: + self.fetch_pk_for_table(table, schema_name, db_name, table_identifier) + + if self.config.include_foreign_keys: + self.fetch_foreign_keys_for_table( + table, schema_name, db_name, table_identifier + ) + + yield from self.gen_dataset_workunits(table, schema_name, db_name) + + def fetch_foreign_keys_for_table( + self, + table: SnowflakeTable, + schema_name: str, + db_name: str, + table_identifier: str, + ) -> None: + try: + table.foreign_keys = self.get_fk_constraints_for_table( + table.name, schema_name, db_name + ) + except Exception as e: + logger.debug( + f"Failed to get foreign key for table {table_identifier} due to error {e}", + exc_info=e, + ) + self.report_warning("Failed to get foreign key for table", table_identifier) + + def fetch_pk_for_table( + self, + table: SnowflakeTable, + schema_name: str, + db_name: str, + table_identifier: str, + ) -> None: + try: + table.pk = self.get_pk_constraints_for_table( + table.name, schema_name, db_name + ) + except Exception as e: + logger.debug( + f"Failed to get primary key for table {table_identifier} due to error {e}", + exc_info=e, + ) + self.report_warning("Failed to get primary key for table", table_identifier) + + def _process_view( + self, + view: SnowflakeView, + snowflake_schema: SnowflakeSchema, + db_name: str, + ) -> Iterable[MetadataWorkUnit]: + schema_name = snowflake_schema.name + view_name = self.get_dataset_identifier(view.name, schema_name, db_name) + + try: + view.columns = self.get_columns_for_table( + view.name, snowflake_schema, db_name + ) + if self.config.extract_tags != TagOption.skip: + view.column_tags = self.tag_extractor.get_column_tags_for_table( + view.name, schema_name, db_name + ) + except Exception as e: + logger.debug( + f"Failed to get columns for view {view_name} due to error {e}", + exc_info=e, + ) + self.report_warning("Failed to get columns for view", view_name) + + if self.config.extract_tags != TagOption.skip: + view.tags = self.tag_extractor.get_tags_on_object( + table_name=view.name, + schema_name=schema_name, + db_name=db_name, + domain="table", + ) + + if self.config.include_technical_schema: + yield from self.gen_dataset_workunits(view, schema_name, db_name) + + def _process_tag(self, tag: SnowflakeTag) -> Iterable[MetadataWorkUnit]: + tag_identifier = tag.identifier() + + if self.report.is_tag_processed(tag_identifier): + return + + self.report.report_tag_processed(tag_identifier) + + yield from self.gen_tag_workunits(tag) + + def gen_dataset_workunits( + self, + table: Union[SnowflakeTable, SnowflakeView], + schema_name: str, + db_name: str, + ) -> Iterable[MetadataWorkUnit]: + if table.tags: + for tag in table.tags: + yield from self._process_tag(tag) + for column_name in table.column_tags: + for tag in table.column_tags[column_name]: + yield from self._process_tag(tag) + + dataset_name = self.get_dataset_identifier(table.name, schema_name, db_name) + dataset_urn = self.dataset_urn_builder(dataset_name) + + status = Status(removed=False) + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=status + ).as_workunit() + + schema_metadata = self.gen_schema_metadata(table, schema_name, db_name) + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=schema_metadata + ).as_workunit() + + dataset_properties = self.get_dataset_properties(table, schema_name, db_name) + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=dataset_properties + ).as_workunit() + + schema_container_key = gen_schema_key( + db_name=self.snowflake_identifier(db_name), + schema=self.snowflake_identifier(schema_name), + platform=self.platform, + platform_instance=self.config.platform_instance, + env=self.config.env, + ) + + yield from add_table_to_schema_container( + dataset_urn=dataset_urn, + parent_container_key=schema_container_key, + ) + dpi_aspect = get_dataplatform_instance_aspect( + dataset_urn=dataset_urn, + platform=self.platform, + platform_instance=self.config.platform_instance, + ) + if dpi_aspect: + yield dpi_aspect + + subTypes = SubTypes( + typeNames=( + [DatasetSubTypes.VIEW] + if isinstance(table, SnowflakeView) + else [DatasetSubTypes.TABLE] + ) + ) + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=subTypes + ).as_workunit() + + if self.domain_registry: + yield from get_domain_wu( + dataset_name=dataset_name, + entity_urn=dataset_urn, + domain_config=self.config.domain, + domain_registry=self.domain_registry, + ) + + if table.tags: + tag_associations = [ + TagAssociation( + tag=make_tag_urn(self.snowflake_identifier(tag.identifier())) + ) + for tag in table.tags + ] + global_tags = GlobalTags(tag_associations) + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=global_tags + ).as_workunit() + + if isinstance(table, SnowflakeView) and table.view_definition is not None: + view_properties_aspect = ViewProperties( + materialized=table.materialized, + viewLanguage="SQL", + viewLogic=table.view_definition, + ) + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=view_properties_aspect + ).as_workunit() + + def get_dataset_properties( + self, + table: Union[SnowflakeTable, SnowflakeView], + schema_name: str, + db_name: str, + ) -> DatasetProperties: + return DatasetProperties( + name=table.name, + created=( + TimeStamp(time=int(table.created.timestamp() * 1000)) + if table.created is not None + else None + ), + lastModified=( + TimeStamp(time=int(table.last_altered.timestamp() * 1000)) + if table.last_altered is not None + else None + ), + description=table.comment, + qualifiedName=f"{db_name}.{schema_name}.{table.name}", + customProperties={}, + externalUrl=( + self.snowsight_url_builder.get_external_url_for_table( + table.name, + schema_name, + db_name, + ( + SnowflakeObjectDomain.TABLE + if isinstance(table, SnowflakeTable) + else SnowflakeObjectDomain.VIEW + ), + ) + if self.snowsight_url_builder + else None + ), + ) + + def gen_tag_workunits(self, tag: SnowflakeTag) -> Iterable[MetadataWorkUnit]: + tag_urn = make_tag_urn(self.snowflake_identifier(tag.identifier())) + + tag_properties_aspect = TagProperties( + name=tag.display_name(), + description=f"Represents the Snowflake tag `{tag._id_prefix_as_str()}` with value `{tag.value}`.", + ) + + yield MetadataChangeProposalWrapper( + entityUrn=tag_urn, aspect=tag_properties_aspect + ).as_workunit() + + def gen_schema_metadata( + self, + table: Union[SnowflakeTable, SnowflakeView], + schema_name: str, + db_name: str, + ) -> SchemaMetadata: + dataset_name = self.get_dataset_identifier(table.name, schema_name, db_name) + dataset_urn = self.dataset_urn_builder(dataset_name) + + foreign_keys: Optional[List[ForeignKeyConstraint]] = None + if isinstance(table, SnowflakeTable) and len(table.foreign_keys) > 0: + foreign_keys = self.build_foreign_keys(table, dataset_urn) + + schema_metadata = SchemaMetadata( + schemaName=dataset_name, + platform=make_data_platform_urn(self.platform), + version=0, + hash="", + platformSchema=MySqlDDL(tableSchema=""), + fields=[ + SchemaField( + fieldPath=self.snowflake_identifier(col.name), + type=SchemaFieldDataType( + SNOWFLAKE_FIELD_TYPE_MAPPINGS.get(col.data_type, NullType)() + ), + # NOTE: nativeDataType will not be in sync with older connector + nativeDataType=col.get_precise_native_type(), + description=col.comment, + nullable=col.is_nullable, + isPartOfKey=( + col.name in table.pk.column_names + if isinstance(table, SnowflakeTable) and table.pk is not None + else None + ), + globalTags=( + GlobalTags( + [ + TagAssociation( + make_tag_urn( + self.snowflake_identifier(tag.identifier()) + ) + ) + for tag in table.column_tags[col.name] + ] + ) + if col.name in table.column_tags + else None + ), + ) + for col in table.columns + ], + foreignKeys=foreign_keys, + ) + + if self.aggregator: + self.aggregator.register_schema(urn=dataset_urn, schema=schema_metadata) + + return schema_metadata + + def build_foreign_keys( + self, table: SnowflakeTable, dataset_urn: str + ) -> List[ForeignKeyConstraint]: + foreign_keys = [] + for fk in table.foreign_keys: + foreign_dataset = make_dataset_urn_with_platform_instance( + platform=self.platform, + name=self.get_dataset_identifier( + fk.referred_table, fk.referred_schema, fk.referred_database + ), + env=self.config.env, + platform_instance=self.config.platform_instance, + ) + foreign_keys.append( + ForeignKeyConstraint( + name=fk.name, + foreignDataset=foreign_dataset, + foreignFields=[ + make_schema_field_urn( + foreign_dataset, + self.snowflake_identifier(col), + ) + for col in fk.referred_column_names + ], + sourceFields=[ + make_schema_field_urn( + dataset_urn, + self.snowflake_identifier(col), + ) + for col in fk.column_names + ], + ) + ) + return foreign_keys + + def gen_database_containers( + self, database: SnowflakeDatabase + ) -> Iterable[MetadataWorkUnit]: + database_container_key = gen_database_key( + self.snowflake_identifier(database.name), + platform=self.platform, + platform_instance=self.config.platform_instance, + env=self.config.env, + ) + + yield from gen_database_container( + name=database.name, + database=self.snowflake_identifier(database.name), + database_container_key=database_container_key, + sub_types=[DatasetContainerSubTypes.DATABASE], + domain_registry=self.domain_registry, + domain_config=self.config.domain, + external_url=( + self.snowsight_url_builder.get_external_url_for_database(database.name) + if self.snowsight_url_builder + else None + ), + description=database.comment, + created=( + int(database.created.timestamp() * 1000) + if database.created is not None + else None + ), + last_modified=( + int(database.last_altered.timestamp() * 1000) + if database.last_altered is not None + else ( + int(database.created.timestamp() * 1000) + if database.created is not None + else None + ) + ), + tags=( + [self.snowflake_identifier(tag.identifier()) for tag in database.tags] + if database.tags + else None + ), + ) + + def gen_schema_containers( + self, schema: SnowflakeSchema, db_name: str + ) -> Iterable[MetadataWorkUnit]: + schema_name = self.snowflake_identifier(schema.name) + database_container_key = gen_database_key( + database=self.snowflake_identifier(db_name), + platform=self.platform, + platform_instance=self.config.platform_instance, + env=self.config.env, + ) + + schema_container_key = gen_schema_key( + db_name=self.snowflake_identifier(db_name), + schema=schema_name, + platform=self.platform, + platform_instance=self.config.platform_instance, + env=self.config.env, + ) + + yield from gen_schema_container( + name=schema.name, + schema=self.snowflake_identifier(schema.name), + database=self.snowflake_identifier(db_name), + database_container_key=database_container_key, + domain_config=self.config.domain, + schema_container_key=schema_container_key, + sub_types=[DatasetContainerSubTypes.SCHEMA], + domain_registry=self.domain_registry, + description=schema.comment, + external_url=( + self.snowsight_url_builder.get_external_url_for_schema( + schema.name, db_name + ) + if self.snowsight_url_builder + else None + ), + created=( + int(schema.created.timestamp() * 1000) + if schema.created is not None + else None + ), + last_modified=( + int(schema.last_altered.timestamp() * 1000) + if schema.last_altered is not None + else None + ), + tags=( + [self.snowflake_identifier(tag.identifier()) for tag in schema.tags] + if schema.tags + else None + ), + ) + + def get_tables_for_schema( + self, schema_name: str, db_name: str + ) -> List[SnowflakeTable]: + tables = self.data_dictionary.get_tables_for_database(db_name) + + # get all tables for database failed, + # falling back to get tables for schema + if tables is None: + self.report.num_get_tables_for_schema_queries += 1 + return self.data_dictionary.get_tables_for_schema(schema_name, db_name) + + # Some schema may not have any table + return tables.get(schema_name, []) + + def get_views_for_schema( + self, schema_name: str, db_name: str + ) -> List[SnowflakeView]: + views = self.data_dictionary.get_views_for_database(db_name) + + # Some schema may not have any table + return views.get(schema_name, []) + + def get_columns_for_table( + self, table_name: str, snowflake_schema: SnowflakeSchema, db_name: str + ) -> List[SnowflakeColumn]: + schema_name = snowflake_schema.name + columns = self.data_dictionary.get_columns_for_schema( + schema_name, + db_name, + cache_exclude_all_objects=itertools.chain( + snowflake_schema.tables, snowflake_schema.views + ), + ) + + # Access to table but none of its columns - is this possible ? + return columns.get(table_name, []) + + def get_pk_constraints_for_table( + self, table_name: str, schema_name: str, db_name: str + ) -> Optional[SnowflakePK]: + constraints = self.data_dictionary.get_pk_constraints_for_schema( + schema_name, db_name + ) + + # Access to table but none of its constraints - is this possible ? + return constraints.get(table_name) + + def get_fk_constraints_for_table( + self, table_name: str, schema_name: str, db_name: str + ) -> List[SnowflakeFK]: + constraints = self.data_dictionary.get_fk_constraints_for_schema( + schema_name, db_name + ) + + # Access to table but none of its constraints - is this possible ? + return constraints.get(table_name, []) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_summary.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_summary.py new file mode 100644 index 0000000000000..f78ae70291f8a --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_summary.py @@ -0,0 +1,138 @@ +import dataclasses +import logging +from collections import defaultdict +from typing import Dict, Iterable, List, Optional + +from datahub.configuration.source_common import LowerCaseDatasetUrnConfigMixin +from datahub.configuration.time_window_config import BaseTimeWindowConfig +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.api.decorators import SupportStatus, config_class, support_status +from datahub.ingestion.api.source import Source, SourceReport +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeFilterConfig +from datahub.ingestion.source.snowflake.snowflake_connection import ( + SnowflakeConnectionConfig, +) +from datahub.ingestion.source.snowflake.snowflake_schema import SnowflakeDatabase +from datahub.ingestion.source.snowflake.snowflake_schema_gen import ( + SnowflakeSchemaGenerator, +) +from datahub.ingestion.source_report.time_window import BaseTimeWindowReport +from datahub.utilities.lossy_collections import LossyList + + +class SnowflakeSummaryConfig( + SnowflakeFilterConfig, + SnowflakeConnectionConfig, + BaseTimeWindowConfig, + LowerCaseDatasetUrnConfigMixin, +): + pass + + +@dataclasses.dataclass +class SnowflakeSummaryReport(SourceReport, BaseTimeWindowReport): + filtered: LossyList[str] = dataclasses.field(default_factory=LossyList) + + num_get_tables_for_schema_queries: int = 0 + num_get_views_for_schema_queries: int = 0 + + schema_counters: Dict[str, int] = dataclasses.field(default_factory=dict) + object_counters: Dict[str, Dict[str, int]] = dataclasses.field( + default_factory=lambda: defaultdict(lambda: defaultdict(int)) + ) + + num_snowflake_queries: Optional[int] = None + num_snowflake_mutations: Optional[int] = None + + def report_dropped(self, ent_name: str) -> None: + self.filtered.append(ent_name) + + def report_entity_scanned(self, name: str, ent_type: str = "table") -> None: + pass + + +@config_class(SnowflakeSummaryConfig) +@support_status(SupportStatus.INCUBATING) +class SnowflakeSummarySource(Source): + def __init__(self, ctx: PipelineContext, config: SnowflakeSummaryConfig): + super().__init__(ctx) + self.config: SnowflakeSummaryConfig = config + self.report: SnowflakeSummaryReport = SnowflakeSummaryReport() + self.logger = logging.getLogger(__name__) + + self.connection = self.config.get_connection() + + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: + schema_generator = SnowflakeSchemaGenerator( + # This is a hack, but we just hope that the config / report have all the fields we need. + config=self.config, # type: ignore + report=self.report, # type: ignore + connection=self.connection, + dataset_urn_builder=lambda x: "", + domain_registry=None, + profiler=None, + aggregator=None, + snowsight_url_builder=None, + ) + + # Databases. + databases: List[SnowflakeDatabase] = [] + for database in schema_generator.get_databases() or []: + # TODO: Support database_patterns. + if not self.config.database_pattern.allowed(database.name): + self.report.report_dropped(f"{database.name}.*") + else: + databases.append(database) + + # Schemas. + for database in databases: + schema_generator.fetch_schemas_for_database(database, database.name) + + self.report.schema_counters[database.name] = len(database.schemas) + + for schema in database.schemas: + # Tables/views. + tables = schema_generator.fetch_tables_for_schema( + schema, database.name, schema.name + ) + views = schema_generator.fetch_views_for_schema( + schema, database.name, schema.name + ) + + self.report.object_counters[database.name][schema.name] = len( + tables + ) + len(views) + + # Queries for usage. + start_time_millis = self.config.start_time.timestamp() * 1000 + end_time_millis = self.config.end_time.timestamp() * 1000 + for row in self.connection.query( + f"""\ +SELECT COUNT(*) AS CNT +FROM snowflake.account_usage.query_history +WHERE query_history.start_time >= to_timestamp_ltz({start_time_millis}, 3) + AND query_history.start_time < to_timestamp_ltz({end_time_millis}, 3) +""" + ): + self.report.num_snowflake_queries = row["CNT"] + + # Queries for lineage/operations. + for row in self.connection.query( + f"""\ +SELECT COUNT(*) AS CNT +FROM + snowflake.account_usage.access_history access_history +WHERE query_start_time >= to_timestamp_ltz({start_time_millis}, 3) + AND query_start_time < to_timestamp_ltz({end_time_millis}, 3) + AND access_history.objects_modified is not null + AND ARRAY_SIZE(access_history.objects_modified) > 0 +""" + ): + self.report.num_snowflake_mutations = row["CNT"] + + # This source doesn't produce any metadata itself. All important information goes into the report. + yield from [] + + def get_report(self) -> SnowflakeSummaryReport: + return self.report diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py index e8b56a01944ad..c5e0994059f2e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py @@ -5,23 +5,22 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple import pydantic -from snowflake.connector import SnowflakeConnection from datahub.configuration.time_window_config import BaseTimeWindowConfig from datahub.emitter.mce_builder import make_user_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.api.closeable import Closeable from datahub.ingestion.api.source_helpers import auto_empty_dataset_usage_statistics from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.snowflake.constants import SnowflakeEdition from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeV2Config -from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery -from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report -from datahub.ingestion.source.snowflake.snowflake_utils import ( - SnowflakeCommonMixin, - SnowflakeConnectionMixin, +from datahub.ingestion.source.snowflake.snowflake_connection import ( + SnowflakeConnection, SnowflakePermissionError, - SnowflakeQueryMixin, ) +from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery +from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report +from datahub.ingestion.source.snowflake.snowflake_utils import SnowflakeCommonMixin from datahub.ingestion.source.state.redundant_run_skip_handler import ( RedundantUsageRunSkipHandler, ) @@ -107,13 +106,12 @@ class SnowflakeJoinedAccessEvent(PermissiveModel): role_name: str -class SnowflakeUsageExtractor( - SnowflakeQueryMixin, SnowflakeConnectionMixin, SnowflakeCommonMixin -): +class SnowflakeUsageExtractor(SnowflakeCommonMixin, Closeable): def __init__( self, config: SnowflakeV2Config, report: SnowflakeV2Report, + connection: SnowflakeConnection, dataset_urn_builder: Callable[[str], str], redundant_run_skip_handler: Optional[RedundantUsageRunSkipHandler], ) -> None: @@ -121,7 +119,7 @@ def __init__( self.report: SnowflakeV2Report = report self.dataset_urn_builder = dataset_urn_builder self.logger = logger - self.connection: Optional[SnowflakeConnection] = None + self.connection = connection self.redundant_run_skip_handler = redundant_run_skip_handler self.start_time, self.end_time = ( @@ -144,11 +142,6 @@ def get_usage_workunits( return self.report.set_ingestion_stage("*", USAGE_EXTRACTION_USAGE_AGGREGATION) - - self.connection = self.create_connection() - if self.connection is None: - return - if self.report.edition == SnowflakeEdition.STANDARD.value: logger.info( "Snowflake Account is Standard Edition. Usage and Operation History Feature is not supported." @@ -207,7 +200,7 @@ def _get_workunits_internal( with PerfTimer() as timer: logger.info("Getting aggregated usage statistics") try: - results = self.query( + results = self.connection.query( SnowflakeQuery.usage_per_object_per_time_bucket_for_time_window( start_time_millis=int(self.start_time.timestamp() * 1000), end_time_millis=int(self.end_time.timestamp() * 1000), @@ -239,7 +232,7 @@ def _get_workunits_internal( logger.debug(f"Processing usage row number {results.rownumber}") logger.debug(self.report.usage_aggregation.as_string()) - if not self._is_dataset_pattern_allowed( + if not self.is_dataset_pattern_allowed( row["OBJECT_NAME"], row["OBJECT_DOMAIN"], ): @@ -293,7 +286,7 @@ def build_usage_statistics_for_dataset( f"Failed to parse usage statistics for dataset {dataset_identifier} due to error {e}.", exc_info=e, ) - self.report_warning( + self.report.warning( "Failed to parse usage statistics for dataset", dataset_identifier ) @@ -376,7 +369,8 @@ def _get_snowflake_history(self) -> Iterable[SnowflakeJoinedAccessEvent]: with PerfTimer() as timer: query = self._make_operations_query() try: - results = self.query(query) + assert self.connection is not None + results = self.connection.query(query) except Exception as e: logger.debug(e, exc_info=e) self.warn_if_stateful_else_error( @@ -398,7 +392,10 @@ def _make_operations_query(self) -> str: def _check_usage_date_ranges(self) -> None: with PerfTimer() as timer: try: - results = self.query(SnowflakeQuery.get_access_history_date_range()) + assert self.connection is not None + results = self.connection.query( + SnowflakeQuery.get_access_history_date_range() + ) except Exception as e: if isinstance(e, SnowflakePermissionError): error_msg = "Failed to get usage. Please grant imported privileges on SNOWFLAKE database. " @@ -407,7 +404,7 @@ def _check_usage_date_ranges(self) -> None: ) else: logger.debug(e, exc_info=e) - self.report_warning( + self.report.warning( "usage", f"Extracting the date range for usage data from Snowflake failed due to error {e}.", ) @@ -419,7 +416,7 @@ def _check_usage_date_ranges(self) -> None: or db_row["MIN_TIME"] is None or db_row["MAX_TIME"] is None ): - self.report_warning( + self.report.warning( "check-usage-data", f"Missing data for access_history {db_row}.", ) @@ -505,7 +502,7 @@ def _process_snowflake_history_row( yield event except Exception as e: self.report.rows_parsing_error += 1 - self.report_warning( + self.report.warning( "operation", f"Failed to parse operation history row {event_dict}, {e}", ) @@ -564,7 +561,7 @@ def _is_unsupported_object_accessed(self, obj: Dict[str, Any]) -> bool: def _is_object_valid(self, obj: Dict[str, Any]) -> bool: if self._is_unsupported_object_accessed( obj - ) or not self._is_dataset_pattern_allowed( + ) or not self.is_dataset_pattern_allowed( obj.get("objectName"), obj.get("objectDomain") ): return False @@ -590,3 +587,6 @@ def _should_ingest_usage(self) -> bool: def report_status(self, step: str, status: bool) -> None: if self.redundant_run_skip_handler: self.redundant_run_skip_handler.report_current_run_status(step, status) + + def close(self) -> None: + pass diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py index 5708b9f168c51..c33fbb3d0bfc8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py @@ -1,53 +1,43 @@ -import logging -from typing import Any, Optional +import abc +from typing import ClassVar, Literal, Optional, Tuple -from snowflake.connector import SnowflakeConnection -from snowflake.connector.cursor import DictCursor from typing_extensions import Protocol -from datahub.configuration.common import MetaError from datahub.configuration.pattern_utils import is_schema_allowed +from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance +from datahub.ingestion.api.source import SourceReport from datahub.ingestion.source.snowflake.constants import ( - GENERIC_PERMISSION_ERROR_KEY, SNOWFLAKE_REGION_CLOUD_REGION_MAPPING, SnowflakeCloudProvider, SnowflakeObjectDomain, ) -from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeV2Config +from datahub.ingestion.source.snowflake.snowflake_config import ( + SnowflakeFilterConfig, + SnowflakeIdentifierConfig, + SnowflakeV2Config, +) from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report -logger: logging.Logger = logging.getLogger(__name__) - - -class SnowflakePermissionError(MetaError): - """A permission error has happened""" - - -# Required only for mypy, since we are using mixin classes, and not inheritance. -# Reference - https://mypy.readthedocs.io/en/latest/more_types.html#mixin-classes -class SnowflakeLoggingProtocol(Protocol): - logger: logging.Logger - -class SnowflakeQueryProtocol(SnowflakeLoggingProtocol, Protocol): - def get_connection(self) -> SnowflakeConnection: +class SnowflakeStructuredReportMixin(abc.ABC): + @property + @abc.abstractmethod + def structured_reporter(self) -> SourceReport: ... + # TODO: Eventually I want to deprecate these methods and use the structured_reporter directly. + def report_warning(self, key: str, reason: str) -> None: + self.structured_reporter.warning(key, reason) -class SnowflakeQueryMixin: - def query(self: SnowflakeQueryProtocol, query: str) -> Any: - try: - self.logger.debug(f"Query : {query}") - resp = self.get_connection().cursor(DictCursor).execute(query) - return resp + def report_error(self, key: str, reason: str) -> None: + self.structured_reporter.failure(key, reason) - except Exception as e: - if is_permission_error(e): - raise SnowflakePermissionError(e) from e - raise +# Required only for mypy, since we are using mixin classes, and not inheritance. +# Reference - https://mypy.readthedocs.io/en/latest/more_types.html#mixin-classes +class SnowflakeCommonProtocol(Protocol): + platform: str = "snowflake" -class SnowflakeCommonProtocol(SnowflakeLoggingProtocol, Protocol): config: SnowflakeV2Config report: SnowflakeV2Report @@ -56,6 +46,9 @@ def get_dataset_identifier( ) -> str: ... + def cleanup_qualified_name(self, qualified_name: str) -> str: + ... + def get_dataset_identifier_from_qualified_name(self, qualified_name: str) -> str: ... @@ -69,10 +62,8 @@ def report_error(self, key: str, reason: str) -> None: ... -class SnowflakeCommonMixin: - platform = "snowflake" - - CLOUD_REGION_IDS_WITHOUT_CLOUD_SUFFIX = [ +class SnowsightUrlBuilder: + CLOUD_REGION_IDS_WITHOUT_CLOUD_SUFFIX: ClassVar = [ "us-west-2", "us-east-1", "eu-west-1", @@ -81,13 +72,21 @@ class SnowflakeCommonMixin: "ap-southeast-2", ] + snowsight_base_url: str + + def __init__(self, account_locator: str, region: str, privatelink: bool = False): + cloud, cloud_region_id = self.get_cloud_region_from_snowflake_region_id(region) + self.snowsight_base_url = self.create_snowsight_base_url( + account_locator, cloud_region_id, cloud, privatelink + ) + @staticmethod def create_snowsight_base_url( account_locator: str, cloud_region_id: str, cloud: str, privatelink: bool = False, - ) -> Optional[str]: + ) -> str: if cloud: url_cloud_provider_suffix = f".{cloud}" @@ -96,7 +95,7 @@ def create_snowsight_base_url( # https://docs.snowflake.com/en/user-guide/admin-account-identifier#non-vps-account-locator-formats-by-cloud-platform-and-region if ( cloud_region_id - in SnowflakeCommonMixin.CLOUD_REGION_IDS_WITHOUT_CLOUD_SUFFIX + in SnowsightUrlBuilder.CLOUD_REGION_IDS_WITHOUT_CLOUD_SUFFIX ): url_cloud_provider_suffix = "" else: @@ -108,7 +107,10 @@ def create_snowsight_base_url( return url @staticmethod - def get_cloud_region_from_snowflake_region_id(region): + def get_cloud_region_from_snowflake_region_id( + region: str, + ) -> Tuple[str, str]: + cloud: str if region in SNOWFLAKE_REGION_CLOUD_REGION_MAPPING.keys(): cloud, cloud_region_id = SNOWFLAKE_REGION_CLOUD_REGION_MAPPING[region] elif region.startswith(("aws_", "gcp_", "azure_")): @@ -119,14 +121,42 @@ def get_cloud_region_from_snowflake_region_id(region): raise Exception(f"Unknown snowflake region {region}") return cloud, cloud_region_id - def _is_dataset_pattern_allowed( - self: SnowflakeCommonProtocol, + # domain is either "view" or "table" + def get_external_url_for_table( + self, + table_name: str, + schema_name: str, + db_name: str, + domain: Literal[SnowflakeObjectDomain.TABLE, SnowflakeObjectDomain.VIEW], + ) -> Optional[str]: + return f"{self.snowsight_base_url}#/data/databases/{db_name}/schemas/{schema_name}/{domain}/{table_name}/" + + def get_external_url_for_schema( + self, schema_name: str, db_name: str + ) -> Optional[str]: + return f"{self.snowsight_base_url}#/data/databases/{db_name}/schemas/{schema_name}/" + + def get_external_url_for_database(self, db_name: str) -> Optional[str]: + return f"{self.snowsight_base_url}#/data/databases/{db_name}/" + + +class SnowflakeFilterMixin(SnowflakeStructuredReportMixin): + @property + @abc.abstractmethod + def filter_config(self) -> SnowflakeFilterConfig: + ... + + @staticmethod + def _combine_identifier_parts( + table_name: str, schema_name: str, db_name: str + ) -> str: + return f"{db_name}.{schema_name}.{table_name}" + + def is_dataset_pattern_allowed( + self, dataset_name: Optional[str], dataset_type: Optional[str], - is_upstream: bool = False, ) -> bool: - if is_upstream and not self.config.validate_upstreams_against_patterns: - return True if not dataset_type or not dataset_name: return True dataset_params = dataset_name.split(".") @@ -145,39 +175,101 @@ def _is_dataset_pattern_allowed( # NOTE: this case returned `True` earlier when extracting lineage return False - if not self.config.database_pattern.allowed( + if not self.filter_config.database_pattern.allowed( dataset_params[0].strip('"') ) or not is_schema_allowed( - self.config.schema_pattern, + self.filter_config.schema_pattern, dataset_params[1].strip('"'), dataset_params[0].strip('"'), - self.config.match_fully_qualified_names, + self.filter_config.match_fully_qualified_names, ): return False if dataset_type.lower() in { SnowflakeObjectDomain.TABLE - } and not self.config.table_pattern.allowed( - self.get_dataset_identifier_from_qualified_name(dataset_name) + } and not self.filter_config.table_pattern.allowed( + self.cleanup_qualified_name(dataset_name) ): return False if dataset_type.lower() in { - "view", - "materialized_view", - } and not self.config.view_pattern.allowed( - self.get_dataset_identifier_from_qualified_name(dataset_name) + SnowflakeObjectDomain.VIEW, + SnowflakeObjectDomain.MATERIALIZED_VIEW, + } and not self.filter_config.view_pattern.allowed( + self.cleanup_qualified_name(dataset_name) ): return False return True - def snowflake_identifier(self: SnowflakeCommonProtocol, identifier: str) -> str: + # Qualified Object names from snowflake audit logs have quotes for for snowflake quoted identifiers, + # For example "test-database"."test-schema".test_table + # whereas we generate urns without quotes even for quoted identifiers for backward compatibility + # and also unavailability of utility function to identify whether current table/schema/database + # name should be quoted in above method get_dataset_identifier + def cleanup_qualified_name(self, qualified_name: str) -> str: + name_parts = qualified_name.split(".") + if len(name_parts) != 3: + self.structured_reporter.report_warning( + title="Unexpected dataset pattern", + message="We failed to parse a Snowflake qualified name into its constituent parts. " + "DB/schema/table filtering may not work as expected on these entities.", + context=f"{qualified_name} has {len(name_parts)} parts", + ) + return qualified_name.replace('"', "") + return SnowflakeFilterMixin._combine_identifier_parts( + table_name=name_parts[2].strip('"'), + schema_name=name_parts[1].strip('"'), + db_name=name_parts[0].strip('"'), + ) + + +class SnowflakeIdentifierMixin(abc.ABC): + platform = "snowflake" + + @property + @abc.abstractmethod + def identifier_config(self) -> SnowflakeIdentifierConfig: + ... + + def snowflake_identifier(self, identifier: str) -> str: # to be in in sync with older connector, convert name to lowercase - if self.config.convert_urns_to_lowercase: + if self.identifier_config.convert_urns_to_lowercase: return identifier.lower() return identifier + def get_dataset_identifier( + self, table_name: str, schema_name: str, db_name: str + ) -> str: + return self.snowflake_identifier( + SnowflakeCommonMixin._combine_identifier_parts( + table_name=table_name, schema_name=schema_name, db_name=db_name + ) + ) + + def gen_dataset_urn(self, dataset_identifier: str) -> str: + return make_dataset_urn_with_platform_instance( + platform=self.platform, + name=dataset_identifier, + platform_instance=self.identifier_config.platform_instance, + env=self.identifier_config.env, + ) + + +# TODO: We're most of the way there on fully removing SnowflakeCommonProtocol. +class SnowflakeCommonMixin(SnowflakeFilterMixin, SnowflakeIdentifierMixin): + @property + def structured_reporter(self: SnowflakeCommonProtocol) -> SourceReport: + return self.report + + @property + def filter_config(self: SnowflakeCommonProtocol) -> SnowflakeFilterConfig: + return self.config + + @property + def identifier_config(self: SnowflakeCommonProtocol) -> SnowflakeIdentifierConfig: + return self.config + @staticmethod def get_quoted_identifier_for_database(db_name): return f'"{db_name}"' @@ -186,34 +278,13 @@ def get_quoted_identifier_for_database(db_name): def get_quoted_identifier_for_schema(db_name, schema_name): return f'"{db_name}"."{schema_name}"' + def get_dataset_identifier_from_qualified_name(self, qualified_name: str) -> str: + return self.snowflake_identifier(self.cleanup_qualified_name(qualified_name)) + @staticmethod def get_quoted_identifier_for_table(db_name, schema_name, table_name): return f'"{db_name}"."{schema_name}"."{table_name}"' - def get_dataset_identifier( - self: SnowflakeCommonProtocol, table_name: str, schema_name: str, db_name: str - ) -> str: - return self.snowflake_identifier(f"{db_name}.{schema_name}.{table_name}") - - # Qualified Object names from snowflake audit logs have quotes for for snowflake quoted identifiers, - # For example "test-database"."test-schema".test_table - # whereas we generate urns without quotes even for quoted identifiers for backward compatibility - # and also unavailability of utility function to identify whether current table/schema/database - # name should be quoted in above method get_dataset_identifier - def get_dataset_identifier_from_qualified_name( - self: SnowflakeCommonProtocol, qualified_name: str - ) -> str: - name_parts = qualified_name.split(".") - if len(name_parts) != 3: - self.report.report_warning( - "invalid-dataset-pattern", - f"Found non-parseable {name_parts} for {qualified_name}", - ) - return self.snowflake_identifier(qualified_name.replace('"', "")) - return self.get_dataset_identifier( - name_parts[2].strip('"'), name_parts[1].strip('"'), name_parts[0].strip('"') - ) - # Note - decide how to construct user urns. # Historically urns were created using part before @ from user's email. # Users without email were skipped from both user entries as well as aggregates. @@ -244,70 +315,3 @@ def warn_if_stateful_else_error( self.report_warning(key, reason) else: self.report_error(key, reason) - - def report_warning(self: SnowflakeCommonProtocol, key: str, reason: str) -> None: - self.report.report_warning(key, reason) - self.logger.warning(f"{key} => {reason}") - - def report_error(self: SnowflakeCommonProtocol, key: str, reason: str) -> None: - self.report.report_failure(key, reason) - self.logger.error(f"{key} => {reason}") - - -class SnowflakeConnectionProtocol(SnowflakeLoggingProtocol, Protocol): - connection: Optional[SnowflakeConnection] - config: SnowflakeV2Config - report: SnowflakeV2Report - - def create_connection(self) -> Optional[SnowflakeConnection]: - ... - - def report_error(self, key: str, reason: str) -> None: - ... - - -class SnowflakeConnectionMixin: - def get_connection(self: SnowflakeConnectionProtocol) -> SnowflakeConnection: - if self.connection is None: - # Ideally this is never called here - self.logger.info("Did you forget to initialize connection for module?") - self.connection = self.create_connection() - - # Connection is already present by the time its used for query - # Every module initializes the connection or fails and returns - assert self.connection is not None - return self.connection - - # If connection succeeds, return connection, else return None and report failure - def create_connection( - self: SnowflakeConnectionProtocol, - ) -> Optional[SnowflakeConnection]: - try: - conn = self.config.get_connection() - except Exception as e: - logger.debug(e, exc_info=e) - if "not granted to this user" in str(e): - self.report_error( - GENERIC_PERMISSION_ERROR_KEY, - f"Failed to connect with snowflake due to error {e}", - ) - else: - logger.debug(e, exc_info=e) - self.report_error( - "snowflake-connection", - f"Failed to connect to snowflake instance due to error {e}.", - ) - return None - else: - return conn - - def close(self: SnowflakeConnectionProtocol) -> None: - if self.connection is not None and not self.connection.is_closed(): - self.connection.close() - - -def is_permission_error(e: Exception) -> bool: - msg = str(e) - # 002003 (02000): SQL compilation error: Database/SCHEMA 'XXXX' does not exist or not authorized. - # Insufficient privileges to operate on database 'XXXX' - return "Insufficient privileges" in msg or "not authorized" in msg diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index 140b702a8b74b..d8eda98da422b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -5,18 +5,8 @@ import os.path import platform from dataclasses import dataclass -from typing import Callable, Dict, Iterable, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union -from snowflake.connector import SnowflakeConnection - -from datahub.configuration.pattern_utils import is_schema_allowed -from datahub.emitter.mce_builder import ( - make_data_platform_urn, - make_dataset_urn_with_platform_instance, - make_schema_field_urn, - make_tag_urn, -) -from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SupportStatus, @@ -36,61 +26,37 @@ TestConnectionReport, ) from datahub.ingestion.api.workunit import MetadataWorkUnit -from datahub.ingestion.glossary.classification_mixin import ( - ClassificationHandler, - classification_workunit_processor, -) -from datahub.ingestion.source.common.subtypes import ( - DatasetContainerSubTypes, - DatasetSubTypes, -) from datahub.ingestion.source.snowflake.constants import ( GENERIC_PERMISSION_ERROR_KEY, - SNOWFLAKE_DATABASE, SnowflakeEdition, - SnowflakeObjectDomain, ) -from datahub.ingestion.source.snowflake.snowflake_config import ( - SnowflakeV2Config, - TagOption, +from datahub.ingestion.source.snowflake.snowflake_assertion import ( + SnowflakeAssertionsHandler, +) +from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeV2Config +from datahub.ingestion.source.snowflake.snowflake_connection import ( + SnowflakeConnection, + SnowflakeConnectionConfig, ) -from datahub.ingestion.source.snowflake.snowflake_data_reader import SnowflakeDataReader from datahub.ingestion.source.snowflake.snowflake_lineage_v2 import ( SnowflakeLineageExtractor, ) from datahub.ingestion.source.snowflake.snowflake_profiler import SnowflakeProfiler from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report from datahub.ingestion.source.snowflake.snowflake_schema import ( - SnowflakeColumn, - SnowflakeDatabase, SnowflakeDataDictionary, - SnowflakeFK, - SnowflakePK, SnowflakeQuery, - SnowflakeSchema, - SnowflakeTable, - SnowflakeTag, - SnowflakeView, +) +from datahub.ingestion.source.snowflake.snowflake_schema_gen import ( + SnowflakeSchemaGenerator, ) from datahub.ingestion.source.snowflake.snowflake_shares import SnowflakeSharesHandler -from datahub.ingestion.source.snowflake.snowflake_tag import SnowflakeTagExtractor from datahub.ingestion.source.snowflake.snowflake_usage_v2 import ( SnowflakeUsageExtractor, ) from datahub.ingestion.source.snowflake.snowflake_utils import ( SnowflakeCommonMixin, - SnowflakeConnectionMixin, - SnowflakePermissionError, - SnowflakeQueryMixin, -) -from datahub.ingestion.source.sql.sql_utils import ( - add_table_to_schema_container, - gen_database_container, - gen_database_key, - gen_schema_container, - gen_schema_key, - get_dataplatform_instance_aspect, - get_domain_wu, + SnowsightUrlBuilder, ) from datahub.ingestion.source.state.profiling_state_handler import ProfilingHandler from datahub.ingestion.source.state.redundant_run_skip_handler import ( @@ -103,83 +69,15 @@ from datahub.ingestion.source.state.stateful_ingestion_base import ( StatefulIngestionSourceBase, ) -from datahub.ingestion.source_config.sql.snowflake import BaseSnowflakeConfig from datahub.ingestion.source_report.ingestion_stage import ( LINEAGE_EXTRACTION, METADATA_EXTRACTION, - PROFILING, ) -from datahub.metadata.com.linkedin.pegasus2avro.common import ( - GlobalTags, - Status, - SubTypes, - TagAssociation, - TimeStamp, -) -from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( - DatasetProperties, - ViewProperties, -) -from datahub.metadata.com.linkedin.pegasus2avro.schema import ( - ArrayType, - BooleanType, - BytesType, - DateType, - ForeignKeyConstraint, - MySqlDDL, - NullType, - NumberType, - RecordType, - SchemaField, - SchemaFieldDataType, - SchemaMetadata, - StringType, - TimeType, -) -from datahub.metadata.com.linkedin.pegasus2avro.tag import TagProperties from datahub.sql_parsing.sql_parsing_aggregator import SqlParsingAggregator from datahub.utilities.registries.domain_registry import DomainRegistry logger: logging.Logger = logging.getLogger(__name__) -# https://docs.snowflake.com/en/sql-reference/intro-summary-data-types.html -SNOWFLAKE_FIELD_TYPE_MAPPINGS = { - "DATE": DateType, - "BIGINT": NumberType, - "BINARY": BytesType, - # 'BIT': BIT, - "BOOLEAN": BooleanType, - "CHAR": NullType, - "CHARACTER": NullType, - "DATETIME": TimeType, - "DEC": NumberType, - "DECIMAL": NumberType, - "DOUBLE": NumberType, - "FIXED": NumberType, - "FLOAT": NumberType, - "INT": NumberType, - "INTEGER": NumberType, - "NUMBER": NumberType, - # 'OBJECT': ? - "REAL": NumberType, - "BYTEINT": NumberType, - "SMALLINT": NumberType, - "STRING": StringType, - "TEXT": StringType, - "TIME": TimeType, - "TIMESTAMP": TimeType, - "TIMESTAMP_TZ": TimeType, - "TIMESTAMP_LTZ": TimeType, - "TIMESTAMP_NTZ": TimeType, - "TINYINT": NumberType, - "VARBINARY": BytesType, - "VARCHAR": StringType, - "VARIANT": RecordType, - "OBJECT": NullType, - "ARRAY": ArrayType, - "GEOGRAPHY": NullType, -} - @platform_name("Snowflake", doc_order=1) @config_class(SnowflakeV2Config) @@ -221,8 +119,6 @@ supported=True, ) class SnowflakeV2Source( - SnowflakeQueryMixin, - SnowflakeConnectionMixin, SnowflakeCommonMixin, StatefulIngestionSourceBase, TestableSource, @@ -232,8 +128,8 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config): self.config: SnowflakeV2Config = config self.report: SnowflakeV2Report = SnowflakeV2Report() self.logger = logger - self.snowsight_base_url: Optional[str] = None - self.connection: Optional[SnowflakeConnection] = None + + self.connection = self.config.get_connection() self.domain_registry: Optional[DomainRegistry] = None if self.config.domain: @@ -242,7 +138,7 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config): ) # For database, schema, tables, views, etc - self.data_dictionary = SnowflakeDataDictionary() + self.data_dictionary = SnowflakeDataDictionary(connection=self.connection) self.lineage_extractor: Optional[SnowflakeLineageExtractor] = None self.aggregator: Optional[SqlParsingAggregator] = None @@ -251,9 +147,22 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config): platform=self.platform, platform_instance=self.config.platform_instance, env=self.config.env, - graph=self.ctx.graph, + graph=( + # If we're ingestion schema metadata for tables/views, then we will populate + # schemas into the resolver as we go. We only need to do a bulk fetch + # if we're not ingesting schema metadata as part of ingestion. + self.ctx.graph + if not ( + self.config.include_technical_schema + and self.config.include_tables + and self.config.include_views + ) + and not self.config.lazy_schema_resolver + else None + ), generate_usage_statistics=False, generate_operations=False, + format_queries=self.config.format_sql_queries, ) self.report.sql_aggregator = self.aggregator.report @@ -270,6 +179,7 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config): self.lineage_extractor = SnowflakeLineageExtractor( config, self.report, + connection=self.connection, dataset_urn_builder=self.gen_dataset_urn, redundant_run_skip_handler=redundant_lineage_run_skip_handler, sql_aggregator=self.aggregator, @@ -290,14 +200,11 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config): self.usage_extractor = SnowflakeUsageExtractor( config, self.report, + connection=self.connection, dataset_urn_builder=self.gen_dataset_urn, redundant_run_skip_handler=redundant_usage_run_skip_handler, ) - self.tag_extractor = SnowflakeTagExtractor( - config, self.data_dictionary, self.report - ) - self.profiling_state_handler: Optional[ProfilingHandler] = None if self.config.enable_stateful_profiling: self.profiling_state_handler = ProfilingHandler( @@ -307,16 +214,13 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config): run_id=self.ctx.run_id, ) + # For profiling + self.profiler: Optional[SnowflakeProfiler] = None if config.is_profiling_enabled(): - # For profiling self.profiler = SnowflakeProfiler( config, self.report, self.profiling_state_handler ) - self.classification_handler = ClassificationHandler(self.config, self.report) - - # Caches tables for a single database. Consider moving to disk or S3 when possible. - self.db_tables: Dict[str, List[SnowflakeTable]] = {} self.add_config_to_report() @classmethod @@ -329,7 +233,9 @@ def test_connection(config_dict: dict) -> TestConnectionReport: test_report = TestConnectionReport() try: - connection_conf = BaseSnowflakeConfig.parse_obj_allow_extras(config_dict) + connection_conf = SnowflakeConnectionConfig.parse_obj_allow_extras( + config_dict + ) connection: SnowflakeConnection = connection_conf.get_connection() assert connection @@ -355,7 +261,7 @@ def test_connection(config_dict: dict) -> TestConnectionReport: @staticmethod def check_capabilities( - conn: SnowflakeConnection, connection_conf: BaseSnowflakeConfig + conn: SnowflakeConnection, connection_conf: SnowflakeConnectionConfig ) -> Dict[Union[SourceCapability, str], CapabilityReport]: # Currently only overall capabilities are reported. # Resource level variations in capabilities are not considered. @@ -366,19 +272,14 @@ class SnowflakePrivilege: object_name: str object_type: str - def query(query): - logger.info(f"Query : {query}") - resp = conn.cursor().execute(query) - return resp - _report: Dict[Union[SourceCapability, str], CapabilityReport] = dict() privileges: List[SnowflakePrivilege] = [] capabilities: List[SourceCapability] = [c.capability for c in SnowflakeV2Source.get_capabilities() if c.capability not in (SourceCapability.PLATFORM_INSTANCE, SourceCapability.DOMAINS, SourceCapability.DELETION_DETECTION)] # type: ignore - cur = query("select current_role()") + cur = conn.query("select current_role()") current_role = [row[0] for row in cur][0] - cur = query("select current_secondary_roles()") + cur = conn.query("select current_secondary_roles()") secondary_roles_str = json.loads([row[0] for row in cur][0])["roles"] secondary_roles = ( [] if secondary_roles_str == "" else secondary_roles_str.split(",") @@ -395,7 +296,7 @@ def query(query): role = roles[i] i = i + 1 # for some roles, quoting is necessary. for example test-role - cur = query(f'show grants to role "{role}"') + cur = conn.query(f'show grants to role "{role}"') for row in cur: privilege = SnowflakePrivilege( privilege=row[1], object_type=row[2], object_name=row[3] @@ -460,7 +361,7 @@ def query(query): ): roles.append(privilege.object_name) - cur = query("select current_warehouse()") + cur = conn.query("select current_warehouse()") current_warehouse = [row[0] for row in cur][0] default_failure_messages = { @@ -522,47 +423,38 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: self._snowflake_clear_ocsp_cache() - self.connection = self.create_connection() + self.connection = self.config.get_connection() if self.connection is None: return - self.inspect_session_metadata() + self.inspect_session_metadata(self.connection) + snowsight_url_builder = None if self.config.include_external_url: - self.snowsight_base_url = self.get_snowsight_base_url() + snowsight_url_builder = self.get_snowsight_url_builder() if self.report.default_warehouse is None: self.report_warehouse_failure() return - self.data_dictionary.set_connection(self.connection) - databases: List[SnowflakeDatabase] = [] - - for database in self.get_databases() or []: - self.report.report_entity_scanned(database.name, "database") - if not self.config.database_pattern.allowed(database.name): - self.report.report_dropped(f"{database.name}.*") - else: - databases.append(database) - - if len(databases) == 0: - return + schema_extractor = SnowflakeSchemaGenerator( + config=self.config, + report=self.report, + connection=self.connection, + domain_registry=self.domain_registry, + profiler=self.profiler, + aggregator=self.aggregator, + snowsight_url_builder=snowsight_url_builder, + dataset_urn_builder=self.gen_dataset_urn, + ) - for snowflake_db in databases: - try: - self.report.set_ingestion_stage(snowflake_db.name, METADATA_EXTRACTION) - yield from self._process_database(snowflake_db) + self.report.set_ingestion_stage("*", METADATA_EXTRACTION) + yield from schema_extractor.get_workunits_internal() - except SnowflakePermissionError as e: - # FIXME - This may break stateful ingestion if new tables than previous run are emitted above - # and stateful ingestion is enabled - self.report_error(GENERIC_PERMISSION_ERROR_KEY, str(e)) - return + databases = schema_extractor.databases self.connection.close() - self.report_cache_info() - # TODO: The checkpoint state for stale entity detection can be committed here. if self.config.shares: @@ -604,16 +496,10 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: ) and self.usage_extractor: yield from self.usage_extractor.get_usage_workunits(discovered_datasets) - def report_cache_info(self) -> None: - lru_cache_functions: List[Callable] = [ - self.data_dictionary.get_tables_for_database, - self.data_dictionary.get_views_for_database, - self.data_dictionary.get_columns_for_schema, - self.data_dictionary.get_pk_constraints_for_schema, - self.data_dictionary.get_fk_constraints_for_schema, - ] - for func in lru_cache_functions: - self.report.lru_cache_info[func.__name__] = func.cache_info()._asdict() # type: ignore + if self.config.include_assertion_results: + yield from SnowflakeAssertionsHandler( + self.config, self.report, self.connection + ).get_assertion_workunits(discovered_datasets) def report_warehouse_failure(self) -> None: if self.config.warehouse is not None: @@ -627,828 +513,9 @@ def report_warehouse_failure(self) -> None: "No default warehouse set for user. Either set default warehouse for user or configure warehouse in recipe.", ) - def get_databases(self) -> Optional[List[SnowflakeDatabase]]: - try: - # `show databases` is required only to get one of the databases - # whose information_schema can be queried to start with. - databases = self.data_dictionary.show_databases() - except Exception as e: - logger.debug(f"Failed to list databases due to error {e}", exc_info=e) - self.report_error( - "list-databases", - f"Failed to list databases due to error {e}", - ) - return None - else: - ischema_databases: List[ - SnowflakeDatabase - ] = self.get_databases_from_ischema(databases) - - if len(ischema_databases) == 0: - self.report_error( - GENERIC_PERMISSION_ERROR_KEY, - "No databases found. Please check permissions.", - ) - return ischema_databases - - def get_databases_from_ischema( - self, databases: List[SnowflakeDatabase] - ) -> List[SnowflakeDatabase]: - ischema_databases: List[SnowflakeDatabase] = [] - for database in databases: - try: - ischema_databases = self.data_dictionary.get_databases(database.name) - break - except Exception: - # query fails if "USAGE" access is not granted for database - # This is okay, because `show databases` query lists all databases irrespective of permission, - # if role has `MANAGE GRANTS` privilege. (not advisable) - logger.debug( - f"Failed to list databases {database.name} information_schema" - ) - # SNOWFLAKE database always shows up even if permissions are missing - if database == SNOWFLAKE_DATABASE: - continue - logger.info( - f"The role {self.report.role} has `MANAGE GRANTS` privilege. This is not advisable and also not required." - ) - - return ischema_databases - - def _process_database( - self, snowflake_db: SnowflakeDatabase - ) -> Iterable[MetadataWorkUnit]: - db_name = snowflake_db.name - - try: - self.query(SnowflakeQuery.use_database(db_name)) - except Exception as e: - if isinstance(e, SnowflakePermissionError): - # This may happen if REFERENCE_USAGE permissions are set - # We can not run show queries on database in such case. - # This need not be a failure case. - self.report_warning( - "Insufficient privileges to operate on database, skipping. Please grant USAGE permissions on database to extract its metadata.", - db_name, - ) - else: - logger.debug( - f"Failed to use database {db_name} due to error {e}", - exc_info=e, - ) - self.report_warning( - "Failed to get schemas for database", - db_name, - ) - return - - if self.config.extract_tags != TagOption.skip: - snowflake_db.tags = self.tag_extractor.get_tags_on_object( - domain="database", db_name=db_name - ) - - if self.config.include_technical_schema: - yield from self.gen_database_containers(snowflake_db) - - self.fetch_schemas_for_database(snowflake_db, db_name) - - if self.config.include_technical_schema and snowflake_db.tags: - for tag in snowflake_db.tags: - yield from self._process_tag(tag) - - self.db_tables = {} - for snowflake_schema in snowflake_db.schemas: - yield from self._process_schema(snowflake_schema, db_name) - - if self.config.is_profiling_enabled() and self.db_tables: - self.report.set_ingestion_stage(snowflake_db.name, PROFILING) - yield from self.profiler.get_workunits(snowflake_db, self.db_tables) - - def fetch_schemas_for_database( - self, snowflake_db: SnowflakeDatabase, db_name: str - ) -> None: - schemas: List[SnowflakeSchema] = [] - try: - for schema in self.data_dictionary.get_schemas_for_database(db_name): - self.report.report_entity_scanned(schema.name, "schema") - if not is_schema_allowed( - self.config.schema_pattern, - schema.name, - db_name, - self.config.match_fully_qualified_names, - ): - self.report.report_dropped(f"{db_name}.{schema.name}.*") - else: - schemas.append(schema) - except Exception as e: - if isinstance(e, SnowflakePermissionError): - error_msg = f"Failed to get schemas for database {db_name}. Please check permissions." - # Ideal implementation would use PEP 678 – Enriching Exceptions with Notes - raise SnowflakePermissionError(error_msg) from e.__cause__ - else: - logger.debug( - f"Failed to get schemas for database {db_name} due to error {e}", - exc_info=e, - ) - self.report_warning( - "Failed to get schemas for database", - db_name, - ) - - if not schemas: - self.report_warning( - "No schemas found in database. If schemas exist, please grant USAGE permissions on them.", - db_name, - ) - else: - snowflake_db.schemas = schemas - - def _process_schema( - self, snowflake_schema: SnowflakeSchema, db_name: str - ) -> Iterable[MetadataWorkUnit]: - schema_name = snowflake_schema.name - - if self.config.extract_tags != TagOption.skip: - snowflake_schema.tags = self.tag_extractor.get_tags_on_object( - schema_name=schema_name, db_name=db_name, domain="schema" - ) - - if self.config.include_technical_schema: - yield from self.gen_schema_containers(snowflake_schema, db_name) - - if self.config.include_tables: - tables = self.fetch_tables_for_schema( - snowflake_schema, db_name, schema_name - ) - self.db_tables[schema_name] = tables - - if self.config.include_technical_schema: - data_reader = self.make_data_reader() - for table in tables: - table_wu_generator = self._process_table( - table, schema_name, db_name - ) - yield from classification_workunit_processor( - table_wu_generator, - self.classification_handler, - data_reader, - [db_name, schema_name, table.name], - ) - - if self.config.include_views: - views = self.fetch_views_for_schema(snowflake_schema, db_name, schema_name) - if ( - self.aggregator - and self.config.include_view_lineage - and self.config.parse_view_ddl - ): - for view in views: - view_identifier = self.get_dataset_identifier( - view.name, schema_name, db_name - ) - if view.view_definition: - self.aggregator.add_view_definition( - view_urn=self.gen_dataset_urn(view_identifier), - view_definition=view.view_definition, - default_db=db_name, - default_schema=schema_name, - ) - - if self.config.include_technical_schema: - for view in views: - yield from self._process_view(view, schema_name, db_name) - - if self.config.include_technical_schema and snowflake_schema.tags: - for tag in snowflake_schema.tags: - yield from self._process_tag(tag) - - if not snowflake_schema.views and not snowflake_schema.tables: - self.report_warning( - "No tables/views found in schema. If tables exist, please grant REFERENCES or SELECT permissions on them.", - f"{db_name}.{schema_name}", - ) - - def fetch_views_for_schema( - self, snowflake_schema: SnowflakeSchema, db_name: str, schema_name: str - ) -> List[SnowflakeView]: - try: - views: List[SnowflakeView] = [] - for view in self.get_views_for_schema(schema_name, db_name): - view_name = self.get_dataset_identifier(view.name, schema_name, db_name) - - self.report.report_entity_scanned(view_name, "view") - - if not self.config.view_pattern.allowed(view_name): - self.report.report_dropped(view_name) - else: - views.append(view) - snowflake_schema.views = [view.name for view in views] - return views - except Exception as e: - if isinstance(e, SnowflakePermissionError): - # Ideal implementation would use PEP 678 – Enriching Exceptions with Notes - error_msg = f"Failed to get views for schema {db_name}.{schema_name}. Please check permissions." - - raise SnowflakePermissionError(error_msg) from e.__cause__ - else: - logger.debug( - f"Failed to get views for schema {db_name}.{schema_name} due to error {e}", - exc_info=e, - ) - self.report_warning( - "Failed to get views for schema", - f"{db_name}.{schema_name}", - ) - return [] - - def fetch_tables_for_schema( - self, snowflake_schema: SnowflakeSchema, db_name: str, schema_name: str - ) -> List[SnowflakeTable]: - try: - tables: List[SnowflakeTable] = [] - for table in self.get_tables_for_schema(schema_name, db_name): - table_identifier = self.get_dataset_identifier( - table.name, schema_name, db_name - ) - self.report.report_entity_scanned(table_identifier) - if not self.config.table_pattern.allowed(table_identifier): - self.report.report_dropped(table_identifier) - else: - tables.append(table) - snowflake_schema.tables = [table.name for table in tables] - return tables - except Exception as e: - if isinstance(e, SnowflakePermissionError): - # Ideal implementation would use PEP 678 – Enriching Exceptions with Notes - error_msg = f"Failed to get tables for schema {db_name}.{schema_name}. Please check permissions." - raise SnowflakePermissionError(error_msg) from e.__cause__ - else: - logger.debug( - f"Failed to get tables for schema {db_name}.{schema_name} due to error {e}", - exc_info=e, - ) - self.report_warning( - "Failed to get tables for schema", - f"{db_name}.{schema_name}", - ) - return [] - - def make_data_reader(self) -> Optional[SnowflakeDataReader]: - if self.classification_handler.is_classification_enabled() and self.connection: - return SnowflakeDataReader.create( - self.connection, self.snowflake_identifier - ) - - return None - - def _process_table( - self, - table: SnowflakeTable, - schema_name: str, - db_name: str, - ) -> Iterable[MetadataWorkUnit]: - table_identifier = self.get_dataset_identifier(table.name, schema_name, db_name) - - self.fetch_columns_for_table(table, schema_name, db_name, table_identifier) - - self.fetch_pk_for_table(table, schema_name, db_name, table_identifier) - - self.fetch_foreign_keys_for_table(table, schema_name, db_name, table_identifier) - - if self.config.extract_tags != TagOption.skip: - table.tags = self.tag_extractor.get_tags_on_object( - table_name=table.name, - schema_name=schema_name, - db_name=db_name, - domain="table", - ) - - if self.config.include_technical_schema: - if table.tags: - for tag in table.tags: - yield from self._process_tag(tag) - for column_name in table.column_tags: - for tag in table.column_tags[column_name]: - yield from self._process_tag(tag) - - yield from self.gen_dataset_workunits(table, schema_name, db_name) - - def fetch_foreign_keys_for_table( - self, - table: SnowflakeTable, - schema_name: str, - db_name: str, - table_identifier: str, - ) -> None: - try: - table.foreign_keys = self.get_fk_constraints_for_table( - table.name, schema_name, db_name - ) - except Exception as e: - logger.debug( - f"Failed to get foreign key for table {table_identifier} due to error {e}", - exc_info=e, - ) - self.report_warning("Failed to get foreign key for table", table_identifier) - - def fetch_pk_for_table( - self, - table: SnowflakeTable, - schema_name: str, - db_name: str, - table_identifier: str, - ) -> None: - try: - table.pk = self.get_pk_constraints_for_table( - table.name, schema_name, db_name - ) - except Exception as e: - logger.debug( - f"Failed to get primary key for table {table_identifier} due to error {e}", - exc_info=e, - ) - self.report_warning("Failed to get primary key for table", table_identifier) - - def fetch_columns_for_table( - self, - table: SnowflakeTable, - schema_name: str, - db_name: str, - table_identifier: str, - ) -> None: - try: - table.columns = self.get_columns_for_table(table.name, schema_name, db_name) - table.column_count = len(table.columns) - if self.config.extract_tags != TagOption.skip: - table.column_tags = self.tag_extractor.get_column_tags_for_table( - table.name, schema_name, db_name - ) - except Exception as e: - logger.debug( - f"Failed to get columns for table {table_identifier} due to error {e}", - exc_info=e, - ) - self.report_warning("Failed to get columns for table", table_identifier) - - def _process_view( - self, - view: SnowflakeView, - schema_name: str, - db_name: str, - ) -> Iterable[MetadataWorkUnit]: - view_name = self.get_dataset_identifier(view.name, schema_name, db_name) - - try: - view.columns = self.get_columns_for_table(view.name, schema_name, db_name) - if self.config.extract_tags != TagOption.skip: - view.column_tags = self.tag_extractor.get_column_tags_for_table( - view.name, schema_name, db_name - ) - except Exception as e: - logger.debug( - f"Failed to get columns for view {view_name} due to error {e}", - exc_info=e, - ) - self.report_warning("Failed to get columns for view", view_name) - - if self.config.extract_tags != TagOption.skip: - view.tags = self.tag_extractor.get_tags_on_object( - table_name=view.name, - schema_name=schema_name, - db_name=db_name, - domain="table", - ) - - if self.config.include_technical_schema: - if view.tags: - for tag in view.tags: - yield from self._process_tag(tag) - for column_name in view.column_tags: - for tag in view.column_tags[column_name]: - yield from self._process_tag(tag) - - yield from self.gen_dataset_workunits(view, schema_name, db_name) - - def _process_tag(self, tag: SnowflakeTag) -> Iterable[MetadataWorkUnit]: - tag_identifier = tag.identifier() - - if self.report.is_tag_processed(tag_identifier): - return - - self.report.report_tag_processed(tag_identifier) - - yield from self.gen_tag_workunits(tag) - - def gen_dataset_urn(self, dataset_identifier: str) -> str: - return make_dataset_urn_with_platform_instance( - platform=self.platform, - name=dataset_identifier, - platform_instance=self.config.platform_instance, - env=self.config.env, - ) - - def gen_dataset_workunits( - self, - table: Union[SnowflakeTable, SnowflakeView], - schema_name: str, - db_name: str, - ) -> Iterable[MetadataWorkUnit]: - dataset_name = self.get_dataset_identifier(table.name, schema_name, db_name) - dataset_urn = self.gen_dataset_urn(dataset_name) - - status = Status(removed=False) - yield MetadataChangeProposalWrapper( - entityUrn=dataset_urn, aspect=status - ).as_workunit() - - schema_metadata = self.gen_schema_metadata(table, schema_name, db_name) - - yield MetadataChangeProposalWrapper( - entityUrn=dataset_urn, aspect=schema_metadata - ).as_workunit() - - dataset_properties = self.get_dataset_properties(table, schema_name, db_name) - - yield MetadataChangeProposalWrapper( - entityUrn=dataset_urn, aspect=dataset_properties - ).as_workunit() - - schema_container_key = gen_schema_key( - db_name=self.snowflake_identifier(db_name), - schema=self.snowflake_identifier(schema_name), - platform=self.platform, - platform_instance=self.config.platform_instance, - env=self.config.env, - ) - - yield from add_table_to_schema_container( - dataset_urn=dataset_urn, - parent_container_key=schema_container_key, - ) - dpi_aspect = get_dataplatform_instance_aspect( - dataset_urn=dataset_urn, - platform=self.platform, - platform_instance=self.config.platform_instance, - ) - if dpi_aspect: - yield dpi_aspect - - subTypes = SubTypes( - typeNames=( - [DatasetSubTypes.VIEW] - if isinstance(table, SnowflakeView) - else [DatasetSubTypes.TABLE] - ) - ) - - yield MetadataChangeProposalWrapper( - entityUrn=dataset_urn, aspect=subTypes - ).as_workunit() - - if self.domain_registry: - yield from get_domain_wu( - dataset_name=dataset_name, - entity_urn=dataset_urn, - domain_config=self.config.domain, - domain_registry=self.domain_registry, - ) - - if table.tags: - tag_associations = [ - TagAssociation( - tag=make_tag_urn(self.snowflake_identifier(tag.identifier())) - ) - for tag in table.tags - ] - global_tags = GlobalTags(tag_associations) - yield MetadataChangeProposalWrapper( - entityUrn=dataset_urn, aspect=global_tags - ).as_workunit() - - if isinstance(table, SnowflakeView) and table.view_definition is not None: - view_properties_aspect = ViewProperties( - materialized=table.materialized, - viewLanguage="SQL", - viewLogic=table.view_definition, - ) - - yield MetadataChangeProposalWrapper( - entityUrn=dataset_urn, aspect=view_properties_aspect - ).as_workunit() - - def get_dataset_properties( - self, - table: Union[SnowflakeTable, SnowflakeView], - schema_name: str, - db_name: str, - ) -> DatasetProperties: - return DatasetProperties( - name=table.name, - created=( - TimeStamp(time=int(table.created.timestamp() * 1000)) - if table.created is not None - else None - ), - lastModified=( - TimeStamp(time=int(table.last_altered.timestamp() * 1000)) - if table.last_altered is not None - else ( - TimeStamp(time=int(table.created.timestamp() * 1000)) - if table.created is not None - else None - ) - ), - description=table.comment, - qualifiedName=f"{db_name}.{schema_name}.{table.name}", - customProperties={}, - externalUrl=( - self.get_external_url_for_table( - table.name, - schema_name, - db_name, - ( - SnowflakeObjectDomain.TABLE - if isinstance(table, SnowflakeTable) - else SnowflakeObjectDomain.VIEW - ), - ) - if self.config.include_external_url - else None - ), - ) - - def gen_tag_workunits(self, tag: SnowflakeTag) -> Iterable[MetadataWorkUnit]: - tag_urn = make_tag_urn(self.snowflake_identifier(tag.identifier())) - - tag_properties_aspect = TagProperties( - name=tag.display_name(), - description=f"Represents the Snowflake tag `{tag._id_prefix_as_str()}` with value `{tag.value}`.", - ) - - yield MetadataChangeProposalWrapper( - entityUrn=tag_urn, aspect=tag_properties_aspect - ).as_workunit() - - def gen_schema_metadata( - self, - table: Union[SnowflakeTable, SnowflakeView], - schema_name: str, - db_name: str, - ) -> SchemaMetadata: - dataset_name = self.get_dataset_identifier(table.name, schema_name, db_name) - dataset_urn = self.gen_dataset_urn(dataset_name) - - foreign_keys: Optional[List[ForeignKeyConstraint]] = None - if isinstance(table, SnowflakeTable) and len(table.foreign_keys) > 0: - foreign_keys = self.build_foreign_keys(table, dataset_urn) - - schema_metadata = SchemaMetadata( - schemaName=dataset_name, - platform=make_data_platform_urn(self.platform), - version=0, - hash="", - platformSchema=MySqlDDL(tableSchema=""), - fields=[ - SchemaField( - fieldPath=self.snowflake_identifier(col.name), - type=SchemaFieldDataType( - SNOWFLAKE_FIELD_TYPE_MAPPINGS.get(col.data_type, NullType)() - ), - # NOTE: nativeDataType will not be in sync with older connector - nativeDataType=col.get_precise_native_type(), - description=col.comment, - nullable=col.is_nullable, - isPartOfKey=( - col.name in table.pk.column_names - if isinstance(table, SnowflakeTable) and table.pk is not None - else None - ), - globalTags=( - GlobalTags( - [ - TagAssociation( - make_tag_urn( - self.snowflake_identifier(tag.identifier()) - ) - ) - for tag in table.column_tags[col.name] - ] - ) - if col.name in table.column_tags - else None - ), - ) - for col in table.columns - ], - foreignKeys=foreign_keys, - ) - - if self.aggregator and self.config.parse_view_ddl: - self.aggregator.register_schema(urn=dataset_urn, schema=schema_metadata) - - return schema_metadata - - def build_foreign_keys( - self, table: SnowflakeTable, dataset_urn: str - ) -> List[ForeignKeyConstraint]: - foreign_keys = [] - for fk in table.foreign_keys: - foreign_dataset = make_dataset_urn_with_platform_instance( - platform=self.platform, - name=self.get_dataset_identifier( - fk.referred_table, fk.referred_schema, fk.referred_database - ), - env=self.config.env, - platform_instance=self.config.platform_instance, - ) - foreign_keys.append( - ForeignKeyConstraint( - name=fk.name, - foreignDataset=foreign_dataset, - foreignFields=[ - make_schema_field_urn( - foreign_dataset, - self.snowflake_identifier(col), - ) - for col in fk.referred_column_names - ], - sourceFields=[ - make_schema_field_urn( - dataset_urn, - self.snowflake_identifier(col), - ) - for col in fk.column_names - ], - ) - ) - return foreign_keys - def get_report(self) -> SourceReport: return self.report - def gen_database_containers( - self, database: SnowflakeDatabase - ) -> Iterable[MetadataWorkUnit]: - database_container_key = gen_database_key( - self.snowflake_identifier(database.name), - platform=self.platform, - platform_instance=self.config.platform_instance, - env=self.config.env, - ) - - yield from gen_database_container( - name=database.name, - database=self.snowflake_identifier(database.name), - database_container_key=database_container_key, - sub_types=[DatasetContainerSubTypes.DATABASE], - domain_registry=self.domain_registry, - domain_config=self.config.domain, - external_url=( - self.get_external_url_for_database(database.name) - if self.config.include_external_url - else None - ), - description=database.comment, - created=( - int(database.created.timestamp() * 1000) - if database.created is not None - else None - ), - last_modified=( - int(database.last_altered.timestamp() * 1000) - if database.last_altered is not None - else ( - int(database.created.timestamp() * 1000) - if database.created is not None - else None - ) - ), - tags=( - [self.snowflake_identifier(tag.identifier()) for tag in database.tags] - if database.tags - else None - ), - ) - - def gen_schema_containers( - self, schema: SnowflakeSchema, db_name: str - ) -> Iterable[MetadataWorkUnit]: - schema_name = self.snowflake_identifier(schema.name) - database_container_key = gen_database_key( - database=self.snowflake_identifier(db_name), - platform=self.platform, - platform_instance=self.config.platform_instance, - env=self.config.env, - ) - - schema_container_key = gen_schema_key( - db_name=self.snowflake_identifier(db_name), - schema=schema_name, - platform=self.platform, - platform_instance=self.config.platform_instance, - env=self.config.env, - ) - - yield from gen_schema_container( - name=schema.name, - schema=self.snowflake_identifier(schema.name), - database=self.snowflake_identifier(db_name), - database_container_key=database_container_key, - domain_config=self.config.domain, - schema_container_key=schema_container_key, - sub_types=[DatasetContainerSubTypes.SCHEMA], - domain_registry=self.domain_registry, - description=schema.comment, - external_url=( - self.get_external_url_for_schema(schema.name, db_name) - if self.config.include_external_url - else None - ), - created=( - int(schema.created.timestamp() * 1000) - if schema.created is not None - else None - ), - last_modified=( - int(schema.last_altered.timestamp() * 1000) - if schema.last_altered is not None - else ( - int(schema.created.timestamp() * 1000) - if schema.created is not None - else None - ) - ), - tags=( - [self.snowflake_identifier(tag.identifier()) for tag in schema.tags] - if schema.tags - else None - ), - ) - - def get_tables_for_schema( - self, schema_name: str, db_name: str - ) -> List[SnowflakeTable]: - tables = self.data_dictionary.get_tables_for_database(db_name) - - # get all tables for database failed, - # falling back to get tables for schema - if tables is None: - self.report.num_get_tables_for_schema_queries += 1 - return self.data_dictionary.get_tables_for_schema(schema_name, db_name) - - # Some schema may not have any table - return tables.get(schema_name, []) - - def get_views_for_schema( - self, schema_name: str, db_name: str - ) -> List[SnowflakeView]: - views = self.data_dictionary.get_views_for_database(db_name) - - # get all views for database failed, - # falling back to get views for schema - if views is None: - self.report.num_get_views_for_schema_queries += 1 - return self.data_dictionary.get_views_for_schema(schema_name, db_name) - - # Some schema may not have any table - return views.get(schema_name, []) - - def get_columns_for_table( - self, table_name: str, schema_name: str, db_name: str - ) -> List[SnowflakeColumn]: - columns = self.data_dictionary.get_columns_for_schema(schema_name, db_name) - - # get all columns for schema failed, - # falling back to get columns for table - if columns is None: - self.report.num_get_columns_for_table_queries += 1 - return self.data_dictionary.get_columns_for_table( - table_name, schema_name, db_name - ) - - # Access to table but none of its columns - is this possible ? - return columns.get(table_name, []) - - def get_pk_constraints_for_table( - self, table_name: str, schema_name: str, db_name: str - ) -> Optional[SnowflakePK]: - constraints = self.data_dictionary.get_pk_constraints_for_schema( - schema_name, db_name - ) - - # Access to table but none of its constraints - is this possible ? - return constraints.get(table_name) - - def get_fk_constraints_for_table( - self, table_name: str, schema_name: str, db_name: str - ) -> List[SnowflakeFK]: - constraints = self.data_dictionary.get_fk_constraints_for_schema( - schema_name, db_name - ) - - # Access to table but none of its constraints - is this possible ? - return constraints.get(table_name, []) - def add_config_to_report(self) -> None: self.report.cleaned_account_id = self.config.get_account() self.report.ignore_start_time_lineage = self.config.ignore_start_time_lineage @@ -1468,22 +535,22 @@ def add_config_to_report(self) -> None: self.config.end_time, ) - def inspect_session_metadata(self) -> None: + def inspect_session_metadata(self, connection: SnowflakeConnection) -> None: try: logger.info("Checking current version") - for db_row in self.query(SnowflakeQuery.current_version()): + for db_row in connection.query(SnowflakeQuery.current_version()): self.report.saas_version = db_row["CURRENT_VERSION()"] except Exception as e: self.report_error("version", f"Error: {e}") try: logger.info("Checking current role") - for db_row in self.query(SnowflakeQuery.current_role()): + for db_row in connection.query(SnowflakeQuery.current_role()): self.report.role = db_row["CURRENT_ROLE()"] except Exception as e: self.report_error("version", f"Error: {e}") try: logger.info("Checking current warehouse") - for db_row in self.query(SnowflakeQuery.current_warehouse()): + for db_row in connection.query(SnowflakeQuery.current_warehouse()): self.report.default_warehouse = db_row["CURRENT_WAREHOUSE()"] except Exception as e: self.report_error("current_warehouse", f"Error: {e}") @@ -1497,33 +564,13 @@ def inspect_session_metadata(self) -> None: except Exception: self.report.edition = None - # domain is either "view" or "table" - def get_external_url_for_table( - self, table_name: str, schema_name: str, db_name: str, domain: str - ) -> Optional[str]: - if self.snowsight_base_url is not None: - return f"{self.snowsight_base_url}#/data/databases/{db_name}/schemas/{schema_name}/{domain}/{table_name}/" - return None - - def get_external_url_for_schema( - self, schema_name: str, db_name: str - ) -> Optional[str]: - if self.snowsight_base_url is not None: - return f"{self.snowsight_base_url}#/data/databases/{db_name}/schemas/{schema_name}/" - return None - - def get_external_url_for_database(self, db_name: str) -> Optional[str]: - if self.snowsight_base_url is not None: - return f"{self.snowsight_base_url}#/data/databases/{db_name}/" - return None - - def get_snowsight_base_url(self) -> Optional[str]: + def get_snowsight_url_builder(self) -> Optional[SnowsightUrlBuilder]: try: # See https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#finding-the-region-and-locator-for-an-account - for db_row in self.query(SnowflakeQuery.current_account()): + for db_row in self.connection.query(SnowflakeQuery.current_account()): account_locator = db_row["CURRENT_ACCOUNT()"] - for db_row in self.query(SnowflakeQuery.current_region()): + for db_row in self.connection.query(SnowflakeQuery.current_region()): region = db_row["CURRENT_REGION()"] self.report.account_locator = account_locator @@ -1533,30 +580,25 @@ def get_snowsight_base_url(self) -> Optional[str]: region = region.split(".")[-1].lower() account_locator = account_locator.lower() - cloud, cloud_region_id = self.get_cloud_region_from_snowflake_region_id( - region - ) - - # For privatelink, account identifier ends with .privatelink - # See https://docs.snowflake.com/en/user-guide/organizations-connect.html#private-connectivity-urls - return self.create_snowsight_base_url( + return SnowsightUrlBuilder( account_locator, - cloud_region_id, - cloud, - self.config.account_id.endswith(".privatelink"), # type:ignore + region, + # For privatelink, account identifier ends with .privatelink + # See https://docs.snowflake.com/en/user-guide/organizations-connect.html#private-connectivity-urls + privatelink=self.config.account_id.endswith(".privatelink"), ) except Exception as e: - self.warn( - self.logger, - "snowsight url", - f"unable to get snowsight base url due to an error -> {e}", + self.report.warning( + title="External URL Generation Failed", + message="We were unable to infer the Snowsight base URL for your Snowflake account. External URLs will not be generated.", + exc=e, ) return None def is_standard_edition(self) -> bool: try: - self.query(SnowflakeQuery.show_tags()) + self.connection.query(SnowflakeQuery.show_tags()) return False except Exception as e: if "Unsupported feature 'TAG'" in str(e): diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index 3091791551827..2527ca6bc76c1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -21,7 +21,7 @@ ) import sqlalchemy.dialects.postgresql.base -from sqlalchemy import create_engine, inspect +from sqlalchemy import create_engine, inspect, log as sqlalchemy_log from sqlalchemy.engine.reflection import Inspector from sqlalchemy.engine.row import LegacyRow from sqlalchemy.exc import ProgrammingError @@ -129,8 +129,6 @@ logger: logging.Logger = logging.getLogger(__name__) -MISSING_COLUMN_INFO = "missing column information" - @dataclass class SQLSourceReport(StaleEntityRemovalSourceReport, ClassificationReportMixin): @@ -536,6 +534,9 @@ def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit if logger.isEnabledFor(logging.DEBUG): # If debug logging is enabled, we also want to echo each SQL query issued. sql_config.options.setdefault("echo", True) + # Patch to avoid duplicate logging + # Known issue with sqlalchemy https://stackoverflow.com/questions/60804288/pycharm-duplicated-log-for-sqlalchemy-echo-true + sqlalchemy_log._add_default_handler = lambda x: None # type: ignore # Extra default SQLAlchemy option for better connection pooling and threading. # https://docs.sqlalchemy.org/en/14/core/pooling.html#sqlalchemy.pool.QueuePool.params.max_overflow @@ -918,7 +919,7 @@ def _get_columns( try: columns = inspector.get_columns(table, schema) if len(columns) == 0: - self.warn(logger, MISSING_COLUMN_INFO, dataset_name) + self.warn(logger, "missing column information", dataset_name) except Exception as e: self.warn( logger, @@ -1255,13 +1256,6 @@ def loop_profiler_requests( logger.debug(f"{dataset_name} has already been seen, skipping...") continue - missing_column_info_warn = self.report.warnings.get(MISSING_COLUMN_INFO) - if ( - missing_column_info_warn is not None - and dataset_name in missing_column_info_warn - ): - continue - (partition, custom_sql) = self.generate_partition_profiler_query( schema, table, self.config.profiling.partition_datetime ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py index dd7592f6a8a5e..93c7025aeee4e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py @@ -8,8 +8,9 @@ from datahub.configuration.common import AllowDenyPattern, ConfigModel from datahub.configuration.source_common import ( - DatasetSourceConfigMixin, + EnvConfigMixin, LowerCaseDatasetUrnConfigMixin, + PlatformInstanceConfigMixin, ) from datahub.configuration.validate_field_removal import pydantic_removed_field from datahub.ingestion.api.incremental_lineage_helper import ( @@ -30,17 +31,7 @@ logger: logging.Logger = logging.getLogger(__name__) -class SQLCommonConfig( - StatefulIngestionConfigBase, - DatasetSourceConfigMixin, - LowerCaseDatasetUrnConfigMixin, - IncrementalLineageConfigMixin, - ClassificationSourceConfigMixin, -): - options: dict = pydantic.Field( - default_factory=dict, - description="Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs.", - ) +class SQLFilterConfig(ConfigModel): # Although the 'table_pattern' enables you to skip everything from certain schemas, # having another option to allow/deny on schema level is an optimization for the case when there is a large number # of schemas that one wants to skip and you want to avoid the time to needlessly fetch those tables only to filter @@ -57,6 +48,32 @@ class SQLCommonConfig( default=AllowDenyPattern.allow_all(), description="Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'", ) + + @pydantic.root_validator(pre=True) + def view_pattern_is_table_pattern_unless_specified( + cls, values: Dict[str, Any] + ) -> Dict[str, Any]: + view_pattern = values.get("view_pattern") + table_pattern = values.get("table_pattern") + if table_pattern and not view_pattern: + logger.info(f"Applying table_pattern {table_pattern} to view_pattern.") + values["view_pattern"] = table_pattern + return values + + +class SQLCommonConfig( + StatefulIngestionConfigBase, + PlatformInstanceConfigMixin, + EnvConfigMixin, + LowerCaseDatasetUrnConfigMixin, + IncrementalLineageConfigMixin, + ClassificationSourceConfigMixin, + SQLFilterConfig, +): + options: dict = pydantic.Field( + default_factory=dict, + description="Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs.", + ) profile_pattern: AllowDenyPattern = Field( default=AllowDenyPattern.allow_all(), description="Regex patterns to filter tables (or specific columns) for profiling during ingestion. Note that only tables allowed by the `table_pattern` will be considered.", @@ -103,17 +120,6 @@ def is_profiling_enabled(self) -> bool: self.profiling.operation_config ) - @pydantic.root_validator(pre=True) - def view_pattern_is_table_pattern_unless_specified( - cls, values: Dict[str, Any] - ) -> Dict[str, Any]: - view_pattern = values.get("view_pattern") - table_pattern = values.get("table_pattern") - if table_pattern and not view_pattern: - logger.info(f"Applying table_pattern {table_pattern} to view_pattern.") - values["view_pattern"] = table_pattern - return values - @pydantic.root_validator(skip_on_failure=True) def ensure_profiling_pattern_is_passed_to_profiling( cls, values: Dict[str, Any] diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py index 968989e2548d1..9c8e475e7b307 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py @@ -92,16 +92,25 @@ def generate_profile_workunits( request for request in requests if request.profile_table_level_only ] for request in table_level_profile_requests: - table_level_profile = DatasetProfile( - timestampMillis=int(datetime.now().timestamp() * 1000), - columnCount=request.table.column_count, - rowCount=request.table.rows_count, - sizeInBytes=request.table.size_in_bytes, - ) - dataset_urn = self.dataset_urn_builder(request.pretty_name) - yield MetadataChangeProposalWrapper( - entityUrn=dataset_urn, aspect=table_level_profile - ).as_workunit() + if ( + request.table.column_count is None + and request.table.rows_count is None + and request.table.size_in_bytes is None + ): + logger.warning( + f"Table {request.pretty_name} has no column count, rows count, or size in bytes. Skipping emitting table level profile." + ) + else: + table_level_profile = DatasetProfile( + timestampMillis=int(datetime.now().timestamp() * 1000), + columnCount=request.table.column_count, + rowCount=request.table.rows_count, + sizeInBytes=request.table.size_in_bytes, + ) + dataset_urn = self.dataset_urn_builder(request.pretty_name) + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=table_level_profile + ).as_workunit() if not ge_profile_requests: return diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py index 7534f1295c528..ae56fb87ee528 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py @@ -52,7 +52,6 @@ if TYPE_CHECKING: from datahub.ingestion.source.ge_data_profiler import GEProfilerRequest -MISSING_COLUMN_INFO = "missing column information" logger: logging.Logger = logging.getLogger(__name__) @@ -133,17 +132,8 @@ def create(cls, config_dict: Dict, ctx: PipelineContext) -> "VerticaSource": return cls(config, ctx) def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]: + yield from super().get_workunits_internal() sql_config = self.config - if logger.isEnabledFor(logging.DEBUG): - # If debug logging is enabled, we also want to echo each SQL query issued. - sql_config.options.setdefault("echo", True) - - # Extra default SQLAlchemy option for better connection pooling and threading. - # https://docs.sqlalchemy.org/en/14/core/pooling.html#sqlalchemy.pool.QueuePool.params.max_overflow - if sql_config.is_profiling_enabled(): - sql_config.options.setdefault( - "max_overflow", sql_config.profiling.max_workers - ) for inspector in self.get_inspectors(): profiler = None @@ -170,11 +160,6 @@ def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit ), ) - if sql_config.include_tables: - yield from self.loop_tables(inspector, schema, sql_config) - - if sql_config.include_views: - yield from self.loop_views(inspector, schema, sql_config) if sql_config.include_projections: yield from self.loop_projections(inspector, schema, sql_config) if sql_config.include_models: @@ -190,6 +175,15 @@ def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit profile_requests, profiler, platform=self.platform ) + def get_identifier( + self, *, schema: str, entity: str, inspector: VerticaInspector, **kwargs: Any + ) -> str: + regular = f"{schema}.{entity}" + if self.config.database: + return f"{self.config.database}.{regular}" + current_database = self.get_db_name(inspector) + return f"{current_database}.{regular}" + def get_database_properties( self, inspector: VerticaInspector, database: str ) -> Optional[Dict[str, str]]: @@ -547,12 +541,7 @@ def loop_profiler_requests( else: logger.debug(f"{dataset_name} has already been seen, skipping...") continue - missing_column_info_warn = self.report.warnings.get(MISSING_COLUMN_INFO) - if ( - missing_column_info_warn is not None - and dataset_name in missing_column_info_warn - ): - continue + (partition, custom_sql) = self.generate_partition_profiler_query( schema, projection, self.config.profiling.partition_datetime ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py b/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py index 9154a555f2309..ee1ccdff781dc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py @@ -6,9 +6,9 @@ import pydantic from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.mcp_builder import entity_supports_aspect from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.ingestion_job_checkpointing_provider_base import JobId -from datahub.ingestion.api.source_helpers import auto_stale_entity_removal from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.state.checkpoint import Checkpoint from datahub.ingestion.source.state.entity_removal_state import GenericCheckpointState @@ -23,9 +23,14 @@ ) from datahub.metadata.schema_classes import StatusClass from datahub.utilities.lossy_collections import LossyList +from datahub.utilities.urns.urn import guess_entity_type logger: logging.Logger = logging.getLogger(__name__) +STATEFUL_INGESTION_IGNORED_ENTITY_TYPES = { + "dataProcessInstance", +} + class StatefulStaleMetadataRemovalConfig(StatefulIngestionConfig): """ @@ -48,10 +53,38 @@ class StatefulStaleMetadataRemovalConfig(StatefulIngestionConfig): @dataclass class StaleEntityRemovalSourceReport(StatefulIngestionReport): soft_deleted_stale_entities: LossyList[str] = field(default_factory=LossyList) + last_state_non_deletable_entities: LossyList[str] = field(default_factory=LossyList) def report_stale_entity_soft_deleted(self, urn: str) -> None: self.soft_deleted_stale_entities.append(urn) + def report_last_state_non_deletable_entities(self, urn: str) -> None: + self.last_state_non_deletable_entities.append(urn) + + +def auto_stale_entity_removal( + stale_entity_removal_handler: "StaleEntityRemovalHandler", + stream: Iterable[MetadataWorkUnit], +) -> Iterable[MetadataWorkUnit]: + """ + Record all entities that are found, and emit removals for any that disappeared in this run. + """ + + for wu in stream: + urn = wu.get_urn() + + if wu.is_primary_source: + entity_type = guess_entity_type(urn) + if entity_type is not None: + stale_entity_removal_handler.add_entity_to_state(entity_type, urn) + else: + stale_entity_removal_handler.add_urn_to_skip(urn) + + yield wu + + # Clean up stale entities. + yield from stale_entity_removal_handler.gen_removed_entity_workunits() + class StaleEntityRemovalHandler( StatefulIngestionUsecaseHandlerBase["GenericCheckpointState"] @@ -272,11 +305,23 @@ def gen_removed_entity_workunits(self) -> Iterable[MetadataWorkUnit]: self.add_entity_to_state("", urn) return + report = self.source.get_report() + assert isinstance(report, StaleEntityRemovalSourceReport) + # Everything looks good, emit the soft-deletion workunits for urn in last_checkpoint_state.get_urns_not_in( type="*", other_checkpoint_state=cur_checkpoint_state ): + entity_type = guess_entity_type(urn) + if ( + entity_type in STATEFUL_INGESTION_IGNORED_ENTITY_TYPES + or not entity_supports_aspect(entity_type, StatusClass) + ): + # If any entity does not support aspect 'status' then skip that entity urn + report.report_last_state_non_deletable_entities(urn) + continue if urn in self._urns_to_skip: + report.report_last_state_non_deletable_entities(urn) logger.debug( f"Not soft-deleting entity {urn} since it is in urns_to_skip" ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py index 9559b8e84b85d..dd2dc3301d80e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/superset.py +++ b/metadata-ingestion/src/datahub/ingestion/source/superset.py @@ -273,11 +273,9 @@ def construct_dashboard_from_api_data(self, dashboard_data): dp.parse(dashboard_data.get("changed_on_utc", "now")).timestamp() * 1000 ) title = dashboard_data.get("dashboard_title", "") - # note: the API does not currently supply created_by usernames due to a bug, but we are required to - # provide a created AuditStamp to comply with ChangeAuditStamp model. For now, I sub in the last - # modified actor urn + # note: the API does not currently supply created_by usernames due to a bug last_modified = ChangeAuditStamps( - created=AuditStamp(time=modified_ts, actor=modified_actor), + created=None, lastModified=AuditStamp(time=modified_ts, actor=modified_actor), ) dashboard_url = f"{self.config.display_uri}{dashboard_data.get('url', '')}" @@ -380,11 +378,9 @@ def construct_chart_from_chart_data(self, chart_data): ) title = chart_data.get("slice_name", "") - # note: the API does not currently supply created_by usernames due to a bug, but we are required to - # provide a created AuditStamp to comply with ChangeAuditStamp model. For now, I sub in the last - # modified actor urn + # note: the API does not currently supply created_by usernames due to a bug last_modified = ChangeAuditStamps( - created=AuditStamp(time=modified_ts, actor=modified_actor), + created=None, lastModified=AuditStamp(time=modified_ts, actor=modified_actor), ) chart_type = chart_type_from_viz_type.get(chart_data.get("viz_type", "")) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py index 1d44fb6122a36..b14a4a8586c7d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau.py @@ -1,5 +1,6 @@ import logging import re +from collections import OrderedDict from dataclasses import dataclass from datetime import datetime from functools import lru_cache @@ -15,7 +16,7 @@ Union, cast, ) -from urllib.parse import urlparse +from urllib.parse import quote, urlparse import dateutil.parser as dp import tableauserverclient as TSC @@ -26,6 +27,7 @@ PersonalAccessTokenAuth, Server, ServerResponseError, + SiteItem, TableauAuth, ) from tableauserverclient.server.endpoint.exceptions import NonXMLResponseError @@ -205,24 +207,31 @@ class TableauConnectionConfig(ConfigModel): def remove_trailing_slash(cls, v): return config_clean.remove_trailing_slashes(v) - def make_tableau_client(self) -> Server: + def get_tableau_auth( + self, site: str + ) -> Union[TableauAuth, PersonalAccessTokenAuth]: # https://tableau.github.io/server-client-python/docs/api-ref#authentication authentication: Union[TableauAuth, PersonalAccessTokenAuth] if self.username and self.password: authentication = TableauAuth( username=self.username, password=self.password, - site_id=self.site, + site_id=site, ) elif self.token_name and self.token_value: authentication = PersonalAccessTokenAuth( - self.token_name, self.token_value, self.site + self.token_name, self.token_value, site ) else: raise ConfigurationError( "Tableau Source: Either username/password or token_name/token_value must be set" ) + return authentication + def make_tableau_client(self, site: str) -> Server: + authentication: Union[ + TableauAuth, PersonalAccessTokenAuth + ] = self.get_tableau_auth(site) try: server = Server( self.connect_uri, @@ -397,6 +406,25 @@ class TableauConfig( "(allows to workaround ingestion errors when pre ingested schema and queries are out of sync)", ) + ingest_multiple_sites: bool = Field( + False, + description="When enabled, ingests multiple sites the user has access to. If the user doesn't have access to the default site, specify an initial site to query in the site property. By default all sites the user has access to will be ingested. You can filter sites with the site_name_pattern property. This flag is currently only supported for Tableau Server. Tableau Cloud is not supported.", + ) + + site_name_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="Filter for specific Tableau sites. " + "By default, all sites will be included in the ingestion. " + "You can both allow and deny sites based on their name using their name, or a Regex pattern. " + "Deny patterns always take precedence over allow patterns. " + "This property is currently only supported for Tableau Server. Tableau Cloud is not supported. ", + ) + + add_site_container: bool = Field( + False, + description="When enabled, sites are added as containers and therefore visible in the folder structure within Datahub.", + ) + # pre = True because we want to take some decision before pydantic initialize the configuration to default values @root_validator(pre=True) def projects_backward_compatibility(cls, values: Dict) -> Dict: @@ -425,6 +453,10 @@ class ProjectKey(ContainerKey): project_id: str +class SiteKey(ContainerKey): + site_id: str + + @dataclass class UsageStat: view_count: int @@ -534,42 +566,29 @@ def __init__( ctx: PipelineContext, ): super().__init__(config, ctx) - self.config: TableauConfig = config self.report: TableauSourceReport = TableauSourceReport() self.server: Optional[Server] = None - self.database_tables: Dict[str, DatabaseTable] = {} - self.tableau_stat_registry: Dict[str, UsageStat] = {} - self.tableau_project_registry: Dict[str, TableauProject] = {} - self.workbook_project_map: Dict[str, str] = {} - self.datasource_project_map: Dict[str, str] = {} - - # This map keeps track of the database server connection hostnames. - self.database_server_hostname_map: Dict[str, str] = {} - # This list keeps track of sheets in workbooks so that we retrieve those - # when emitting sheets. - self.sheet_ids: List[str] = [] - # This list keeps track of dashboards in workbooks so that we retrieve those - # when emitting dashboards. - self.dashboard_ids: List[str] = [] - # This list keeps track of embedded datasources in workbooks so that we retrieve those - # when emitting embedded data sources. - self.embedded_datasource_ids_being_used: List[str] = [] - # This list keeps track of datasource being actively used by workbooks so that we only retrieve those - # when emitting published data sources. - self.datasource_ids_being_used: List[str] = [] - # This list keeps track of datasource being actively used by workbooks so that we only retrieve those - # when emitting custom SQL data sources. - self.custom_sql_ids_being_used: List[str] = [] + self._authenticate(self.config.site) - self._authenticate() + def _authenticate(self, site_content_url: str) -> None: + try: + logger.info(f"Authenticated to Tableau site: '{site_content_url}'") + self.server = self.config.make_tableau_client(site_content_url) + # Note that we're not catching ConfigurationError, since we want that to throw. + except ValueError as e: + self.report.failure( + title="Tableau Login Error", + message="Failed to authenticate with Tableau.", + exc=e, + ) @staticmethod def test_connection(config_dict: dict) -> TestConnectionReport: test_report = TestConnectionReport() try: source_config = TableauConfig.parse_obj_allow_extras(config_dict) - source_config.make_tableau_client() + source_config.make_tableau_client(source_config.site) test_report.basic_connectivity = CapabilityReport(capable=True) except Exception as e: test_report.basic_connectivity = CapabilityReport( @@ -577,6 +596,65 @@ def test_connection(config_dict: dict) -> TestConnectionReport: ) return test_report + def get_report(self) -> TableauSourceReport: + return self.report + + @classmethod + def create(cls, config_dict: dict, ctx: PipelineContext) -> Source: + config = TableauConfig.parse_obj(config_dict) + return cls(config, ctx) + + def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: + return [ + *super().get_workunit_processors(), + StaleEntityRemovalHandler.create( + self, self.config, self.ctx + ).workunit_processor, + ] + + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: + if self.server is None or not self.server.is_signed_in(): + return + try: + if self.config.ingest_multiple_sites: + for site in list(TSC.Pager(self.server.sites)): + if ( + site.state != "Active" + or not self.config.site_name_pattern.allowed(site.name) + ): + logger.info( + f"Skip site '{site.name}' as it's excluded in site_name_pattern or inactive." + ) + continue + self.server.auth.switch_site(site) + site_source = TableauSiteSource( + config=self.config, + ctx=self.ctx, + site=site, + report=self.report, + server=self.server, + platform=self.platform, + ) + logger.info(f"Ingesting assets of site '{site.content_url}'.") + yield from site_source.ingest_tableau_site() + else: + site = self.server.sites.get_by_id(self.server.site_id) + site_source = TableauSiteSource( + config=self.config, + ctx=self.ctx, + site=site, + report=self.report, + server=self.server, + platform=self.platform, + ) + yield from site_source.ingest_tableau_site() + except MetadataQueryException as md_exception: + self.report.failure( + title="Failed to Retrieve Tableau Metadata", + message="Unable to retrieve metadata from tableau.", + context=str(md_exception), + ) + def close(self) -> None: try: if self.server is not None: @@ -590,6 +668,48 @@ def close(self) -> None: self.server = None super().close() + +class TableauSiteSource: + def __init__( + self, + config: TableauConfig, + ctx: PipelineContext, + site: SiteItem, + report: TableauSourceReport, + server: Server, + platform: str, + ): + self.config: TableauConfig = config + self.report = report + self.server: Server = server + self.ctx: PipelineContext = ctx + self.site: SiteItem = site + self.platform = platform + + self.database_tables: Dict[str, DatabaseTable] = {} + self.tableau_stat_registry: Dict[str, UsageStat] = {} + self.tableau_project_registry: Dict[str, TableauProject] = {} + self.workbook_project_map: Dict[str, str] = {} + self.datasource_project_map: Dict[str, str] = {} + + # This map keeps track of the database server connection hostnames. + self.database_server_hostname_map: Dict[str, str] = {} + # This list keeps track of sheets in workbooks so that we retrieve those + # when emitting sheets. + self.sheet_ids: List[str] = [] + # This list keeps track of dashboards in workbooks so that we retrieve those + # when emitting dashboards. + self.dashboard_ids: List[str] = [] + # This list keeps track of embedded datasources in workbooks so that we retrieve those + # when emitting embedded data sources. + self.embedded_datasource_ids_being_used: List[str] = [] + # This list keeps track of datasource being actively used by workbooks so that we only retrieve those + # when emitting published data sources. + self.datasource_ids_being_used: List[str] = [] + # This list keeps track of datasource being actively used by workbooks so that we only retrieve those + # when emitting custom SQL data sources. + self.custom_sql_ids_being_used: List[str] = [] + @property def no_env_browse_prefix(self) -> str: # Prefix to use with browse path (v1) @@ -600,13 +720,26 @@ def no_env_browse_prefix(self) -> str: if self.config.platform_instance else self.platform ) - return f"/{platform_with_instance}" + return f"/{platform_with_instance}{self.site_name_browse_path}" + + @property + def site_name_browse_path(self) -> str: + site_name_prefix = ( + self.site.name if self.site and self.config.add_site_container else "" + ) + return f"/{site_name_prefix}" if site_name_prefix else "" @property def dataset_browse_prefix(self) -> str: # datasets also have the env in the browse path return f"/{self.config.env.lower()}{self.no_env_browse_prefix}" + def _re_authenticate(self): + tableau_auth: Union[ + TableauAuth, PersonalAccessTokenAuth + ] = self.config.get_tableau_auth(self.site.content_url) + self.server.auth.sign_in(tableau_auth) + def _populate_usage_stat_registry(self) -> None: if self.server is None: return @@ -665,6 +798,12 @@ def form_path(project_id: str) -> List[str]: cur_proj = all_project_map[project_id] ancestors = [cur_proj.name] while cur_proj.parent_id is not None: + if cur_proj.parent_id not in all_project_map: + self.report.report_warning( + "project-issue", + f"Parent project {cur_proj.parent_id} not found. We need Site Administrator Explorer permissions.", + ) + break cur_proj = all_project_map[cur_proj.parent_id] ancestors = [cur_proj.name, *ancestors] return ancestors @@ -707,7 +846,7 @@ def _is_denied_project(self, project: TableauProject) -> bool: def _init_tableau_project_registry(self, all_project_map: dict) -> None: list_of_skip_projects: List[TableauProject] = [] - + projects_to_ingest = {} for project in all_project_map.values(): # Skip project if it is not allowed logger.debug(f"Evaluating project pattern for {project.name}") @@ -716,7 +855,13 @@ def _init_tableau_project_registry(self, all_project_map: dict) -> None: logger.debug(f"Project {project.name} is skipped") continue logger.debug(f"Project {project.name} is added in project registry") - self.tableau_project_registry[project.id] = project + projects_to_ingest[project.id] = project + + # We rely on automatic browse paths (v2) when creating containers. That's why we need to sort the projects here. + # Otherwise, nested projects will not have the correct browse paths if not created in correct order / hierarchy. + self.tableau_project_registry = OrderedDict( + sorted(projects_to_ingest.items(), key=lambda item: len(item[1].path)) + ) if self.config.extract_project_hierarchy is False: logger.debug( @@ -787,17 +932,6 @@ def _populate_projects_registry(self) -> None: f"Tableau workbooks {self.workbook_project_map}", ) - def _authenticate(self) -> None: - try: - self.server = self.config.make_tableau_client() - logger.info("Authenticated to Tableau server") - # Note that we're not catching ConfigurationError, since we want that to throw. - except ValueError as e: - self.report.failure( - key="tableau-login", - reason=str(e), - ) - def get_data_platform_instance(self) -> DataPlatformInstanceClass: return DataPlatformInstanceClass( platform=builder.make_data_platform_urn(self.platform), @@ -838,7 +972,7 @@ def get_connection_object_page( # If ingestion has been running for over 2 hours, the Tableau # temporary credentials will expire. If this happens, this exception # will be thrown and we need to re-authenticate and retry. - self._authenticate() + self._re_authenticate() return self.get_connection_object_page( query, connection_type, @@ -875,7 +1009,10 @@ def get_connection_object_page( error and (error.get(c.EXTENSIONS) or {}).get(c.SEVERITY) == c.WARNING for error in errors ): - self.report.warning(key=connection_type, reason=f"{errors}") + self.report.warning( + message=f"Received error fetching Query Connection {connection_type}", + context=f"Errors: {errors}", + ) else: raise RuntimeError(f"Query {connection_type} error: {errors}") @@ -1417,6 +1554,7 @@ def emit_custom_sql_datasources(self) -> Iterable[MetadataWorkUnit]: datasource_name = None project = None + columns: List[Dict[Any, Any]] = [] if len(csql[c.DATA_SOURCES]) > 0: # CustomSQLTable id owned by exactly one tableau data source logger.debug( @@ -1464,7 +1602,7 @@ def emit_custom_sql_datasources(self) -> Iterable[MetadataWorkUnit]: project = self._get_project_browse_path_name(datasource) # if condition is needed as graphQL return "columns": None - columns: List[Dict[Any, Any]] = ( + columns = ( cast(List[Dict[Any, Any]], csql.get(c.COLUMNS)) if c.COLUMNS in csql and csql.get(c.COLUMNS) is not None else [] @@ -2308,7 +2446,7 @@ def emit_sheets_as_charts( last_modified = self.get_last_modified(creator, created_at, updated_at) if sheet.get(c.PATH): - site_part = f"/site/{self.config.site}" if self.config.site else "" + site_part = f"/site/{self.site.content_url}" if self.site else "" sheet_external_url = ( f"{self.config.connect_uri}/#{site_part}/views/{sheet.get(c.PATH)}" ) @@ -2319,9 +2457,9 @@ def emit_sheets_as_charts( and sheet[c.CONTAINED_IN_DASHBOARDS][0].get(c.PATH) ): # sheet contained in dashboard - site_part = f"/t/{self.config.site}" if self.config.site else "" + site_part = f"/t/{self.site.content_url}" if self.site else "" dashboard_path = sheet[c.CONTAINED_IN_DASHBOARDS][0][c.PATH] - sheet_external_url = f"{self.config.connect_uri}{site_part}/authoring/{dashboard_path}/{sheet.get(c.NAME, '')}" + sheet_external_url = f"{self.config.connect_uri}{site_part}/authoring/{dashboard_path}/{quote(sheet.get(c.NAME, ''), safe='')}" else: # hidden or viz-in-tooltip sheet sheet_external_url = None @@ -2451,7 +2589,7 @@ def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUni else None ) - site_part = f"/site/{self.config.site}" if self.config.site else "" + site_part = f"/site/{self.site.content_url}" if self.site else "" workbook_uri = workbook.get("uri") workbook_part = ( workbook_uri[workbook_uri.index("/workbooks/") :] if workbook_uri else None @@ -2500,6 +2638,13 @@ def gen_project_key(self, project_luid: str) -> ProjectKey: project_id=project_luid, ) + def gen_site_key(self, site_id: str) -> SiteKey: + return SiteKey( + platform=self.platform, + instance=self.config.platform_instance, + site_id=site_id, + ) + @staticmethod def _create_datahub_dashboard_usage_stat( usage_stat: UsageStat, @@ -2603,7 +2748,7 @@ def emit_dashboard( updated_at = dashboard.get(c.UPDATED_AT, datetime.now()) last_modified = self.get_last_modified(creator, created_at, updated_at) - site_part = f"/site/{self.config.site}" if self.config.site else "" + site_part = f"/site/{self.site.content_url}" if self.site else "" dashboard_external_url = ( f"{self.config.connect_uri}/#{site_part}/views/{dashboard.get(c.PATH, '')}" ) @@ -2746,23 +2891,20 @@ def _get_ownership(self, user: str) -> Optional[OwnershipClass]: return None - @classmethod - def create(cls, config_dict: dict, ctx: PipelineContext) -> Source: - config = TableauConfig.parse_obj(config_dict) - return cls(config, ctx) - def emit_project_containers(self) -> Iterable[MetadataWorkUnit]: for _id, project in self.tableau_project_registry.items(): + parent_container_key: Optional[ContainerKey] = None + if project.parent_id: + parent_container_key = self.gen_project_key(project.parent_id) + elif self.config.add_site_container and self.site and self.site.id: + parent_container_key = self.gen_site_key(self.site.id) + yield from gen_containers( container_key=self.gen_project_key(_id), name=project.name, description=project.description, sub_types=[c.PROJECT], - parent_container_key=( - self.gen_project_key(project.parent_id) - if project.parent_id - else None - ), + parent_container_key=parent_container_key, ) if ( project.parent_id is not None @@ -2777,48 +2919,42 @@ def emit_project_containers(self) -> Iterable[MetadataWorkUnit]: sub_types=[c.PROJECT], ) - def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: - return [ - *super().get_workunit_processors(), - StaleEntityRemovalHandler.create( - self, self.config, self.ctx - ).workunit_processor, - ] - - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: - if self.server is None or not self.server.is_signed_in(): + def emit_site_container(self): + if not self.site or not self.site.id: + logger.warning("Can not ingest site container. No site information found.") return - try: - # Initialise the dictionary to later look-up for chart and dashboard stat - if self.config.extract_usage_stats: - self._populate_usage_stat_registry() - - # Populate the map of database names and database hostnames to be used later to map - # databases to platform instances. - if self.config.database_hostname_to_platform_instance_map: - self._populate_database_server_hostname_map() - - self._populate_projects_registry() - yield from self.emit_project_containers() - yield from self.emit_workbooks() - if self.sheet_ids: - yield from self.emit_sheets() - if self.dashboard_ids: - yield from self.emit_dashboards() - if self.embedded_datasource_ids_being_used: - yield from self.emit_embedded_datasources() - if self.datasource_ids_being_used: - yield from self.emit_published_datasources() - if self.custom_sql_ids_being_used: - yield from self.emit_custom_sql_datasources() - if self.database_tables: - yield from self.emit_upstream_tables() - except MetadataQueryException as md_exception: - self.report.failure( - key="tableau-metadata", - reason=f"Unable to retrieve metadata from tableau. Information: {str(md_exception)}", - ) - def get_report(self) -> TableauSourceReport: - return self.report - return self.report + yield from gen_containers( + container_key=self.gen_site_key(self.site.id), + name=self.site.name or "Default", + sub_types=[c.SITE], + ) + + def ingest_tableau_site(self): + # Initialise the dictionary to later look-up for chart and dashboard stat + if self.config.extract_usage_stats: + self._populate_usage_stat_registry() + + # Populate the map of database names and database hostnames to be used later to map + # databases to platform instances. + if self.config.database_hostname_to_platform_instance_map: + self._populate_database_server_hostname_map() + + self._populate_projects_registry() + + if self.config.add_site_container: + yield from self.emit_site_container() + yield from self.emit_project_containers() + yield from self.emit_workbooks() + if self.sheet_ids: + yield from self.emit_sheets() + if self.dashboard_ids: + yield from self.emit_dashboards() + if self.embedded_datasource_ids_being_used: + yield from self.emit_embedded_datasources() + if self.datasource_ids_being_used: + yield from self.emit_published_datasources() + if self.custom_sql_ids_being_used: + yield from self.emit_custom_sql_datasources() + if self.database_tables: + yield from self.emit_upstream_tables() diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py index 6c75876e68787..93f2a0ef2f6a8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py @@ -767,7 +767,12 @@ def get_overridden_info( ): platform_instance = database_hostname_to_platform_instance_map.get(hostname) - if original_platform in ("athena", "hive", "mysql"): # Two tier databases + if original_platform in ( + "athena", + "hive", + "mysql", + "teradata", + ): # Two tier databases upstream_db = None return upstream_db, platform_instance, platform, original_platform diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau_constant.py b/metadata-ingestion/src/datahub/ingestion/source/tableau_constant.py index 9ead9a407d957..5acadf8ca19e5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau_constant.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau_constant.py @@ -78,4 +78,5 @@ DASHBOARDS_CONNECTION = "dashboardsConnection" EMBEDDED_DATA_SOURCES_CONNECTION = "embeddedDatasourcesConnection" PROJECT = "Project" +SITE = "Site" IS_UNSUPPORTED_CUSTOM_SQL = "isUnsupportedCustomSql" diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py index 1e90f3a044f42..112acd8101297 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py @@ -122,7 +122,7 @@ def catalogs(self, metastore: Optional[Metastore]) -> Iterable[Catalog]: response = self._workspace_client.catalogs.list() if not response: logger.info("Catalogs not found") - return [] + return for catalog in response: optional_catalog = self._create_catalog(metastore, catalog) if optional_catalog: @@ -151,7 +151,7 @@ def schemas(self, catalog: Catalog) -> Iterable[Schema]: response = self._workspace_client.schemas.list(catalog_name=catalog.name) if not response: logger.info(f"Schemas not found for catalog {catalog.id}") - return [] + return for schema in response: optional_schema = self._create_schema(catalog, schema) if optional_schema: @@ -170,7 +170,7 @@ def tables(self, schema: Schema) -> Iterable[Table]: ) if not response: logger.info(f"Tables not found for schema {schema.id}") - return [] + return for table in response: try: optional_table = self._create_table( diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py index 42ca9af7e8459..b29170cb2d705 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py @@ -262,7 +262,7 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: self.report.report_ingestion_stage_start("Ingestion Setup") wait_on_warehouse = None - if self.config.is_profiling_enabled() or self.config.include_hive_metastore: + if self.config.include_hive_metastore: self.report.report_ingestion_stage_start("Start warehouse") # Can take several minutes, so start now and wait later wait_on_warehouse = self.unity_catalog_api_proxy.start_warehouse() @@ -309,9 +309,20 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: ) if self.config.is_profiling_enabled(): - self.report.report_ingestion_stage_start("Wait on warehouse") - assert wait_on_warehouse - wait_on_warehouse.result() + self.report.report_ingestion_stage_start("Start warehouse") + # Need to start the warehouse again for profiling, + # as it may have been stopped after ingestion might take + # longer time to complete + wait_on_warehouse = self.unity_catalog_api_proxy.start_warehouse() + if wait_on_warehouse is None: + self.report.report_failure( + "initialization", + f"SQL warehouse {self.config.profiling.warehouse_id} not found", + ) + return + else: + # wait until warehouse is started + wait_on_warehouse.result() self.report.report_ingestion_stage_start("Profiling") if isinstance(self.config.profiling, UnityCatalogAnalyzeProfilerConfig): diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/clickhouse_usage.py b/metadata-ingestion/src/datahub/ingestion/source/usage/clickhouse_usage.py index 50cbbded69f19..4c2e4d42c440e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/usage/clickhouse_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/usage/clickhouse_usage.py @@ -116,7 +116,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: access_events = self._get_clickhouse_history() # If the query results is empty, we don't want to proceed if not access_events: - return [] + return joined_access_event = self._get_joined_access_event(access_events) aggregated_info = self._aggregate_access_events(joined_access_event) diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/starburst_trino_usage.py b/metadata-ingestion/src/datahub/ingestion/source/usage/starburst_trino_usage.py index 10cf981abce56..6ded11027c83a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/usage/starburst_trino_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/usage/starburst_trino_usage.py @@ -135,7 +135,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: access_events = self._get_trino_history() # If the query results is empty, we don't want to proceed if not access_events: - return [] + return joined_access_event = self._get_joined_access_event(access_events) aggregated_info = self._aggregate_access_events(joined_access_event) diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/operation_config.py b/metadata-ingestion/src/datahub/ingestion/source_config/operation_config.py index 3deb9e75d97e7..a670173aa4751 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_config/operation_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source_config/operation_config.py @@ -69,10 +69,10 @@ def is_profiling_enabled(operation_config: OperationConfig) -> bool: today = datetime.date.today() if ( operation_config.profile_day_of_week is not None - and operation_config.profile_date_of_month != today.weekday() + and operation_config.profile_day_of_week != today.weekday() ): logger.info( - "Profiling won't be done because weekday does not match config profile_date_of_month.", + "Profiling won't be done because weekday does not match config profile_day_of_week.", ) return False if ( diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/usage/snowflake_usage.py b/metadata-ingestion/src/datahub/ingestion/source_config/usage/snowflake_usage.py index 747bde0a8b563..a0e79f62240ee 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_config/usage/snowflake_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source_config/usage/snowflake_usage.py @@ -11,7 +11,7 @@ class SnowflakeUsageConfig(BaseUsageConfig): email_domain: Optional[str] = pydantic.Field( default=None, - description="Email domain of your organisation so users can be displayed on UI appropriately.", + description="Email domain of your organization so users can be displayed on UI appropriately.", ) apply_view_usage_to_tables: bool = pydantic.Field( default=False, diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/auto_helper_transformer.py b/metadata-ingestion/src/datahub/ingestion/transformer/auto_helper_transformer.py new file mode 100644 index 0000000000000..cec7be36060b7 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/transformer/auto_helper_transformer.py @@ -0,0 +1,79 @@ +from typing import Callable, Iterable, Optional, Union + +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.api.common import ControlRecord, PipelineContext, RecordEnvelope +from datahub.ingestion.api.transform import Transformer +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.metadata.com.linkedin.pegasus2avro.mxe import ( + MetadataChangeEvent, + MetadataChangeProposal, +) + + +class AutoHelperTransformer(Transformer): + """Converts an auto_* source helper into a transformer. + + Important usage note: this assumes that the auto helper is stateless. The converter + will be called multiple times, once for each batch of records. If the helper + attempts to maintain state or perform some cleanup at the end of the stream, it + will not behave correctly. + """ + + def __init__( + self, + converter: Callable[[Iterable[MetadataWorkUnit]], Iterable[MetadataWorkUnit]], + ): + self.converter = converter + + def transform( + self, record_envelopes: Iterable[RecordEnvelope] + ) -> Iterable[RecordEnvelope]: + records = list(record_envelopes) + + normal_records = [r for r in records if not isinstance(r.record, ControlRecord)] + control_records = [r for r in records if isinstance(r.record, ControlRecord)] + + yield from self._from_workunits( + self.converter( + self._into_workunits(normal_records), + ) + ) + + # Pass through control records as-is. Note that this isn't fully correct, since it technically + # reorders the control records relative to the normal records. This is ok since the only control + # record we have marks the end of the stream. + yield from control_records + + @classmethod + def _into_workunits( + cls, + stream: Iterable[ + RecordEnvelope[ + Union[ + MetadataChangeEvent, + MetadataChangeProposal, + MetadataChangeProposalWrapper, + ] + ] + ], + ) -> Iterable[MetadataWorkUnit]: + for record in stream: + workunit_id: Optional[str] = record.metadata.get("workunit_id") + metadata = record.record + yield MetadataWorkUnit.from_metadata(metadata, id=workunit_id) + + @classmethod + def _from_workunits( + cls, stream: Iterable[MetadataWorkUnit] + ) -> Iterable[RecordEnvelope]: + for workunit in stream: + yield RecordEnvelope( + workunit.metadata, + { + "workunit_id": workunit.id, + }, + ) + + @classmethod + def create(cls, config_dict: dict, ctx: PipelineContext) -> Transformer: + raise NotImplementedError(f"{cls.__name__} cannot be created from config") diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py b/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py index a78a79141e8e4..3e313ddd356be 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py @@ -27,6 +27,16 @@ def entity_types(self) -> List[str]: return ["dataset"] +class TagTransformer(BaseTransformer, SingleAspectTransformer, metaclass=ABCMeta): + """Transformer that does transform sequentially on each tag.""" + + def __init__(self): + super().__init__() + + def entity_types(self) -> List[str]: + return ["dataset", "container"] + + class DatasetOwnershipTransformer(DatasetTransformer, metaclass=ABCMeta): def aspect_name(self) -> str: return "ownership" @@ -128,3 +138,8 @@ def aspect_name(self) -> str: class DatasetUsageStatisticsTransformer(DatasetTransformer, metaclass=ABCMeta): def aspect_name(self) -> str: return "datasetUsageStatistics" + + +class TagsToTermTransformer(TagTransformer, metaclass=ABCMeta): + def aspect_name(self) -> str: + return "glossaryTerms" diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/system_metadata_transformer.py b/metadata-ingestion/src/datahub/ingestion/transformer/system_metadata_transformer.py new file mode 100644 index 0000000000000..8a0636f5a11a9 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/transformer/system_metadata_transformer.py @@ -0,0 +1,45 @@ +import functools +from typing import Iterable + +from datahub.emitter.mce_builder import get_sys_time +from datahub.ingestion.api.common import PipelineContext, RecordEnvelope +from datahub.ingestion.api.transform import Transformer +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.transformer.auto_helper_transformer import AutoHelperTransformer +from datahub.metadata.schema_classes import SystemMetadataClass + + +def auto_system_metadata( + ctx: PipelineContext, + stream: Iterable[MetadataWorkUnit], +) -> Iterable[MetadataWorkUnit]: + if not ctx.pipeline_config: + raise ValueError("Pipeline config is required for system metadata") + set_system_metadata = ctx.pipeline_config.flags.set_system_metadata + set_pipeline_name = ctx.pipeline_config.flags.set_system_metadata_pipeline_name + + for workunit in stream: + if set_system_metadata: + workunit.metadata.systemMetadata = SystemMetadataClass( + lastObserved=get_sys_time(), runId=ctx.run_id + ) + if set_pipeline_name: + workunit.metadata.systemMetadata.pipelineName = ctx.pipeline_name + + yield workunit + + +class SystemMetadataTransformer(Transformer): + def __init__(self, ctx: PipelineContext): + self._inner_transfomer = AutoHelperTransformer( + functools.partial(auto_system_metadata, ctx) + ) + + def transform( + self, record_envelopes: Iterable[RecordEnvelope] + ) -> Iterable[RecordEnvelope]: + yield from self._inner_transfomer.transform(record_envelopes) + + @classmethod + def create(cls, config_dict: dict, ctx: PipelineContext) -> Transformer: + raise NotImplementedError(f"{cls.__name__} cannot be created from config") diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/tags_to_terms.py b/metadata-ingestion/src/datahub/ingestion/transformer/tags_to_terms.py new file mode 100644 index 0000000000000..338f191c0829d --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/transformer/tags_to_terms.py @@ -0,0 +1,145 @@ +from typing import List, Optional, Set, cast + +import datahub.emitter.mce_builder as builder +from datahub.configuration.common import ( + TransformerSemantics, + TransformerSemanticsConfigModel, +) +from datahub.emitter.mce_builder import Aspect, make_term_urn +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.graph.client import DataHubGraph +from datahub.ingestion.transformer.dataset_transformer import TagsToTermTransformer +from datahub.metadata.schema_classes import ( + AuditStampClass, + GlobalTagsClass, + GlossaryTermAssociationClass, + GlossaryTermsClass, + SchemaMetadataClass, +) + + +class TagsToTermMapperConfig(TransformerSemanticsConfigModel): + tags: List[str] + + +class TagsToTermMapper(TagsToTermTransformer): + """This transformer maps specified tags to corresponding glossary terms for a dataset.""" + + def __init__(self, config: TagsToTermMapperConfig, ctx: PipelineContext): + super().__init__() + self.ctx: PipelineContext = ctx + self.config: TagsToTermMapperConfig = config + + @classmethod + def create(cls, config_dict: dict, ctx: PipelineContext) -> "TagsToTermMapper": + config = TagsToTermMapperConfig.parse_obj(config_dict) + return cls(config, ctx) + + @staticmethod + def _merge_with_server_glossary_terms( + graph: DataHubGraph, + urn: str, + glossary_terms_aspect: Optional[GlossaryTermsClass], + ) -> Optional[GlossaryTermsClass]: + if not glossary_terms_aspect or not glossary_terms_aspect.terms: + # nothing to add, no need to consult server + return None + + # Merge the transformed terms with existing server terms. + # The transformed terms takes precedence, which may change the term context. + server_glossary_terms_aspect = graph.get_glossary_terms(entity_urn=urn) + if server_glossary_terms_aspect is not None: + glossary_terms_aspect.terms = list( + { + **{term.urn: term for term in server_glossary_terms_aspect.terms}, + **{term.urn: term for term in glossary_terms_aspect.terms}, + }.values() + ) + + return glossary_terms_aspect + + @staticmethod + def get_tags_from_global_tags(global_tags: Optional[GlobalTagsClass]) -> Set[str]: + """Extracts tags urn from GlobalTagsClass.""" + if not global_tags or not global_tags.tags: + return set() + + return {tag_assoc.tag for tag_assoc in global_tags.tags} + + @staticmethod + def get_tags_from_schema_metadata( + schema_metadata: Optional[SchemaMetadataClass], + ) -> Set[str]: + """Extracts globalTags from all fields in SchemaMetadataClass.""" + if not schema_metadata or not schema_metadata.fields: + return set() + tags = set() + for field in schema_metadata.fields: + if field.globalTags: + tags.update( + TagsToTermMapper.get_tags_from_global_tags(field.globalTags) + ) + return tags + + def transform_aspect( + self, entity_urn: str, aspect_name: str, aspect: Optional[Aspect] + ) -> Optional[Aspect]: + + in_glossary_terms: Optional[GlossaryTermsClass] = cast( + Optional[GlossaryTermsClass], aspect + ) + + assert self.ctx.graph + in_global_tags_aspect: Optional[GlobalTagsClass] = self.ctx.graph.get_tags( + entity_urn + ) + in_schema_metadata_aspect: Optional[ + SchemaMetadataClass + ] = self.ctx.graph.get_schema_metadata(entity_urn) + + if in_global_tags_aspect is None and in_schema_metadata_aspect is None: + return cast(Aspect, in_glossary_terms) + + global_tags = TagsToTermMapper.get_tags_from_global_tags(in_global_tags_aspect) + schema_metadata_tags = TagsToTermMapper.get_tags_from_schema_metadata( + in_schema_metadata_aspect + ) + + # Combine tags from both global and schema level + combined_tags = global_tags.union(schema_metadata_tags) + + tag_set = set(self.config.tags) + terms_to_add = set() + tags_to_delete = set() + + # Check each global tag against the configured tag list and prepare terms + for full_tag in combined_tags: + tag_name = full_tag.split("urn:li:tag:")[-1].split(".")[-1].split(":")[0] + if tag_name in tag_set: + term_urn = make_term_urn(tag_name) + terms_to_add.add(term_urn) + tags_to_delete.add(full_tag) # Full URN for deletion + + if not terms_to_add: + return cast(Aspect, in_glossary_terms) # No new terms to add + + for tag_urn in tags_to_delete: + self.ctx.graph.remove_tag(tag_urn=tag_urn, resource_urn=entity_urn) + + # Initialize the Glossary Terms properly + out_glossary_terms = GlossaryTermsClass( + terms=[GlossaryTermAssociationClass(urn=term) for term in terms_to_add], + auditStamp=AuditStampClass( + time=builder.get_sys_time(), actor="urn:li:corpUser:restEmitter" + ), + ) + + if self.config.semantics == TransformerSemantics.PATCH: + patch_glossary_terms: Optional[ + GlossaryTermsClass + ] = TagsToTermMapper._merge_with_server_glossary_terms( + self.ctx.graph, entity_urn, out_glossary_terms + ) + return cast(Optional[Aspect], patch_glossary_terms) + else: + return cast(Aspect, out_glossary_terms) diff --git a/metadata-ingestion/src/datahub/integrations/assertion/__init__.py b/metadata-ingestion/src/datahub/integrations/assertion/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/metadata-ingestion/src/datahub/integrations/assertion/common.py b/metadata-ingestion/src/datahub/integrations/assertion/common.py new file mode 100644 index 0000000000000..9ffad5cf66640 --- /dev/null +++ b/metadata-ingestion/src/datahub/integrations/assertion/common.py @@ -0,0 +1,61 @@ +from functools import lru_cache +from typing import List, Optional, Tuple, TypedDict + +from datahub.api.entities.assertion.assertion import BaseEntityAssertion +from datahub.ingestion.graph.client import get_default_graph +from datahub.metadata.com.linkedin.pegasus2avro.dataset import DatasetProperties +from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaMetadata +from datahub.utilities.urns.urn import Urn + + +class ColumnDict(TypedDict): + col: str + native_type: str + + +@lru_cache +def get_qualified_name_from_datahub(urn: str) -> Optional[str]: + with get_default_graph() as graph: + props: Optional[DatasetProperties] = graph.get_aspect(urn, DatasetProperties) + if props is not None: + return props.qualifiedName + return None + + +@lru_cache +def get_schema_from_datahub(urn: str) -> Optional[List[ColumnDict]]: + with get_default_graph() as graph: + schema: Optional[SchemaMetadata] = graph.get_aspect(urn, SchemaMetadata) + if schema is not None: + return [ + {"col": field.fieldPath, "native_type": field.nativeDataType} + for field in schema.fields + ] + return None + + +def get_entity_name(assertion: BaseEntityAssertion) -> Tuple[str, str, str]: + if assertion.meta and assertion.meta.get("entity_qualified_name"): + parts = assertion.meta["entity_qualified_name"].split(".") + else: + qualified_name = get_qualified_name_from_datahub(assertion.entity) + if qualified_name is not None: + parts = qualified_name.split(".") + else: + urn_id = Urn.create_from_string(assertion.entity).entity_ids[1] + parts = urn_id.split(".") + if len(parts) > 3: + parts = parts[-3:] + assert len(parts) == 3 + database = parts[-3] + schema = parts[-2] + table = parts[-1] + return database, schema, table + + +def get_entity_schema(assertion: BaseEntityAssertion) -> Optional[List[ColumnDict]]: + if assertion.meta and assertion.meta.get("entity_schema"): + return assertion.meta.get("entity_schema") + elif get_schema_from_datahub(assertion.entity): + return get_schema_from_datahub(assertion.entity) + return None diff --git a/metadata-ingestion/src/datahub/integrations/assertion/registry.py b/metadata-ingestion/src/datahub/integrations/assertion/registry.py new file mode 100644 index 0000000000000..26015ddbf9a31 --- /dev/null +++ b/metadata-ingestion/src/datahub/integrations/assertion/registry.py @@ -0,0 +1,8 @@ +from typing import Dict, Type + +from datahub.api.entities.assertion.compiler_interface import AssertionCompiler +from datahub.integrations.assertion.snowflake.compiler import SnowflakeAssertionCompiler + +ASSERTION_PLATFORMS: Dict[str, Type[AssertionCompiler]] = { + "snowflake": SnowflakeAssertionCompiler +} diff --git a/metadata-ingestion/src/datahub/integrations/assertion/snowflake/__init__.py b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/metadata-ingestion/src/datahub/integrations/assertion/snowflake/compiler.py b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/compiler.py new file mode 100644 index 0000000000000..8d2ae2960ebd0 --- /dev/null +++ b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/compiler.py @@ -0,0 +1,237 @@ +import logging +import os +from pathlib import Path +from typing import Dict, Tuple + +from datahub.api.entities.assertion.assertion_config_spec import AssertionsConfigSpec +from datahub.api.entities.assertion.assertion_operator import LessThanOrEqualToOperator +from datahub.api.entities.assertion.assertion_trigger import ( + AssertionTrigger, + CronTrigger, + EntityChangeTrigger, + IntervalTrigger, +) +from datahub.api.entities.assertion.compiler_interface import ( + AssertionCompilationResult, + AssertionCompiler, + CompileResultArtifact, + CompileResultArtifactType, +) +from datahub.api.entities.assertion.datahub_assertion import DataHubAssertion +from datahub.api.entities.assertion.field_assertion import FieldValuesAssertion +from datahub.api.entities.assertion.freshness_assertion import ( + FixedIntervalFreshnessAssertion, +) +from datahub.emitter.mce_builder import make_assertion_urn +from datahub.integrations.assertion.common import get_entity_name, get_entity_schema +from datahub.integrations.assertion.snowflake.dmf_generator import SnowflakeDMFHandler +from datahub.integrations.assertion.snowflake.field_metric_sql_generator import ( + SnowflakeFieldMetricSQLGenerator, +) +from datahub.integrations.assertion.snowflake.field_values_metric_sql_generator import ( + SnowflakeFieldValuesMetricSQLGenerator, +) +from datahub.integrations.assertion.snowflake.metric_operator_sql_generator import ( + SnowflakeMetricEvalOperatorSQLGenerator, +) +from datahub.integrations.assertion.snowflake.metric_sql_generator import ( + SnowflakeMetricSQLGenerator, +) + +logger = logging.Logger(__name__) + +DMF_DEFINITIONS_FILE_NAME = "dmf_definitions.sql" +DMF_ASSOCIATIONS_FILE_NAME = "dmf_associations.sql" +DMF_SCHEMA_PROPERTY_KEY = "DMF_SCHEMA" + + +class SnowflakeAssertionCompiler(AssertionCompiler): + def __init__(self, output_dir: str, extras: Dict[str, str]) -> None: + self.output_dir = Path(output_dir) + self.extras = extras + self.metric_generator = SnowflakeMetricSQLGenerator( + SnowflakeFieldMetricSQLGenerator(), SnowflakeFieldValuesMetricSQLGenerator() + ) + self.metric_evaluator = SnowflakeMetricEvalOperatorSQLGenerator() + self.dmf_handler = SnowflakeDMFHandler() + + self._entity_schedule_history: Dict[str, AssertionTrigger] = dict() + + @classmethod + def create( + cls, output_dir: str, extras: Dict[str, str] + ) -> "SnowflakeAssertionCompiler": + assert os.path.exists( + output_dir + ), f"Specified location {output_dir} does not exist." + + assert os.path.isdir( + output_dir + ), f"Specified location {output_dir} is not a folder." + + assert any( + x.upper() == DMF_SCHEMA_PROPERTY_KEY for x in extras + ), "Must specify value for DMF schema using -x DMF_SCHEMA=" + + return SnowflakeAssertionCompiler(output_dir, extras) + + def compile( + self, assertion_config_spec: AssertionsConfigSpec + ) -> AssertionCompilationResult: + result = AssertionCompilationResult("snowflake", "success") + + # TODO: Create/Report permissions sql + + dmf_definitions_path = self.output_dir / DMF_DEFINITIONS_FILE_NAME + dmf_associations_path = self.output_dir / DMF_ASSOCIATIONS_FILE_NAME + with (dmf_definitions_path).open("w") as definitions, ( + dmf_associations_path + ).open("w") as associations: + for assertion_spec in assertion_config_spec.assertions: + result.report.num_processed += 1 + try: + start_line = f"\n-- Start of Assertion {assertion_spec.get_id()}\n" + (dmf_definition, dmf_association) = self.process_assertion( + assertion_spec + ) + end_line = f"\n-- End of Assertion {assertion_spec.get_id()}\n" + + definitions.write(start_line) + definitions.write(dmf_definition) + definitions.write(end_line) + + associations.write(start_line) + associations.write(dmf_association) + associations.write(end_line) + + result.report.num_compile_succeeded += 1 + except Exception as e: + result.status = "failure" + result.report.report_failure( + assertion_spec.get_id(), + f"Failed to compile assertion of type {assertion_spec.assertion.type} due to error: {e}", + ) + result.report.num_compile_failed += 1 + if result.report.num_compile_succeeded > 0: + result.add_artifact( + CompileResultArtifact( + name=DMF_DEFINITIONS_FILE_NAME, + path=dmf_definitions_path, + type=CompileResultArtifactType.SQL_QUERIES, + description="SQL file containing DMF create definitions equivalent to Datahub Assertions", + ) + ) + result.add_artifact( + CompileResultArtifact( + name=DMF_ASSOCIATIONS_FILE_NAME, + path=dmf_associations_path, + type=CompileResultArtifactType.SQL_QUERIES, + description="ALTER TABLE queries to associate DMFs to table to run on configured schedule.", + ) + ) + + return result + + def process_assertion(self, assertion: DataHubAssertion) -> Tuple[str, str]: + # TODO: support schema assertion ? + + # For freshness assertion, metric is difference in seconds between assertion execution time + # and last time table was updated. + # For field values assertion, metric is number or percentage of rows that do not satify + # operator condition. + # For remaining assertions, numeric metric is discernible in assertion definition itself. + metric_definition = self.metric_generator.metric_sql(assertion.assertion) + + if isinstance(assertion.assertion, FixedIntervalFreshnessAssertion): + assertion_sql = self.metric_evaluator.operator_sql( + LessThanOrEqualToOperator( + type="less_than_or_equal_to", + value=assertion.assertion.lookback_interval.total_seconds(), + ), + metric_definition, + ) + elif isinstance(assertion.assertion, FieldValuesAssertion): + assertion_sql = self.metric_evaluator.operator_sql( + LessThanOrEqualToOperator( + type="less_than_or_equal_to", + value=assertion.assertion.failure_threshold.value, + ), + metric_definition, + ) + else: + assertion_sql = self.metric_evaluator.operator_sql( + assertion.assertion.operator, metric_definition + ) + + dmf_name = get_dmf_name(assertion) + dmf_schema_name = self.extras[DMF_SCHEMA_PROPERTY_KEY] + + args_create_dmf, args_add_dmf = get_dmf_args(assertion) + + entity_name = get_entity_name(assertion.assertion) + + self._entity_schedule_history.setdefault( + assertion.assertion.entity, assertion.assertion.trigger + ) + if ( + assertion.assertion.entity in self._entity_schedule_history + and self._entity_schedule_history[assertion.assertion.entity] + != assertion.assertion.trigger + ): + raise ValueError( + "Assertions on same entity must have same schedules as of now." + f" Found different schedules on entity {assertion.assertion.entity} ->" + f" ({self._entity_schedule_history[assertion.assertion.entity].trigger})," + f" ({assertion.assertion.trigger.trigger})" + ) + + dmf_schedule = get_dmf_schedule(assertion.assertion.trigger) + dmf_definition = self.dmf_handler.create_dmf( + f"{dmf_schema_name}.{dmf_name}", + args_create_dmf, + assertion.assertion.description + or f"Created via DataHub for assertion {make_assertion_urn(assertion.get_id())} of type {assertion.assertion.type}", + assertion_sql, + ) + dmf_association = self.dmf_handler.add_dmf_to_table( + f"{dmf_schema_name}.{dmf_name}", + args_add_dmf, + dmf_schedule, + ".".join(entity_name), + ) + + return dmf_definition, dmf_association + + +def get_dmf_name(assertion: DataHubAssertion) -> str: + return f"datahub__{assertion.get_id()}" + + +def get_dmf_args(assertion: DataHubAssertion) -> Tuple[str, str]: + """Returns Tuple with + - Args used to create DMF + - Args used to add DMF to table""" + # Snowflake does not allow creating custom data metric + # function without column name argument. + # So we fetch any one column from table's schema + args_create_dmf = "ARGT TABLE({col_name} {col_type})" + args_add_dmf = "{col_name}" + entity_schema = get_entity_schema(assertion.assertion) + if entity_schema: + for col_dict in entity_schema: + return args_create_dmf.format( + col_name=col_dict["col"], col_type=col_dict["native_type"] + ), args_add_dmf.format(col_name=col_dict["col"]) + + raise ValueError("entity schema not available") + + +def get_dmf_schedule(trigger: AssertionTrigger) -> str: + if isinstance(trigger.trigger, EntityChangeTrigger): + return "TRIGGER_ON_CHANGES" + elif isinstance(trigger.trigger, CronTrigger): + return f"USING CRON {trigger.trigger.cron} {trigger.trigger.timezone}" + elif isinstance(trigger.trigger, IntervalTrigger): + return f"{trigger.trigger.interval.seconds/60} MIN" + else: + raise ValueError(f"Unsupported trigger type {type(trigger.trigger)}") diff --git a/metadata-ingestion/src/datahub/integrations/assertion/snowflake/dmf_generator.py b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/dmf_generator.py new file mode 100644 index 0000000000000..4f50b7c2b81a5 --- /dev/null +++ b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/dmf_generator.py @@ -0,0 +1,22 @@ +class SnowflakeDMFHandler: + def create_dmf( + self, dmf_name: str, dmf_args: str, dmf_comment: str, dmf_sql: str + ) -> str: + return f""" + CREATE or REPLACE DATA METRIC FUNCTION + {dmf_name} ({dmf_args}) + RETURNS NUMBER + COMMENT = '{dmf_comment}' + AS + $$ + {dmf_sql} + $$; + """ + + def add_dmf_to_table( + self, dmf_name: str, dmf_col_args: str, dmf_schedule: str, table_identifier: str + ) -> str: + return f""" + ALTER TABLE {table_identifier} SET DATA_METRIC_SCHEDULE = '{dmf_schedule}'; + ALTER TABLE {table_identifier} ADD DATA METRIC FUNCTION {dmf_name} ON ({dmf_col_args}); + """ diff --git a/metadata-ingestion/src/datahub/integrations/assertion/snowflake/field_metric_sql_generator.py b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/field_metric_sql_generator.py new file mode 100644 index 0000000000000..3ff218a9f280b --- /dev/null +++ b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/field_metric_sql_generator.py @@ -0,0 +1,154 @@ +from typing import List, Optional + +from datahub.api.entities.assertion.field_assertion import FieldMetricAssertion +from datahub.api.entities.assertion.field_metric import FieldMetric +from datahub.integrations.assertion.common import get_entity_name + + +class SnowflakeFieldMetricSQLGenerator: + def unique_count_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + return f"""select count(distinct {field_name}) + from {entity_name} {self._setup_where_clause([dataset_filter])}""" + + def unique_percentage_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + return f"""select count(distinct {field_name})/count(*) + from {entity_name} {self._setup_where_clause([dataset_filter])}""" + + def null_count_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + where_clause = self._setup_where_clause( + [dataset_filter, f"{field_name} is null"] + ) + return f"""select count(*) + from {entity_name} {where_clause}""" + + def null_percentage_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + return f"""select ({self.null_count_sql(field_name, entity_name, dataset_filter)})/count(*) + from {entity_name} {self._setup_where_clause([dataset_filter])}""" + + def min_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + return f"""select min({field_name}) + from {entity_name} {self._setup_where_clause([dataset_filter])}""" + + def max_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + return f"""select max({field_name}) + from {entity_name} {self._setup_where_clause([dataset_filter])}""" + + def mean_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + return f"""select avg({field_name}) + from {entity_name} {self._setup_where_clause([dataset_filter])}""" + + def median_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + return f"""select median({field_name}) + from {entity_name} {self._setup_where_clause([dataset_filter])}""" + + def stddev_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + return f"""select stddev({field_name}) + from {entity_name} {self._setup_where_clause([dataset_filter])}""" + + def negative_count_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + where_clause = self._setup_where_clause([dataset_filter, f"{field_name} < 0"]) + return f"""select count(*) + from {entity_name} {where_clause}""" + + def negative_percentage_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + return f"""select ({self.negative_count_sql(field_name, entity_name, dataset_filter)})/count(*) + from {entity_name} {self._setup_where_clause([dataset_filter])}""" + + def zero_count_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + where_clause = self._setup_where_clause([dataset_filter, f"{field_name} = 0"]) + return f"""select count(*) + from {entity_name} {where_clause}""" + + def zero_percentage_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + return f"""select ({self.zero_count_sql(field_name, entity_name, dataset_filter)})/count(*) + from {entity_name} {self._setup_where_clause([dataset_filter])}""" + + def min_length_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + return f"""select min(length({field_name})) + from {entity_name} {self._setup_where_clause([dataset_filter])}""" + + def max_length_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + return f"""select max(length({field_name})) + from {entity_name} {self._setup_where_clause([dataset_filter])}""" + + def empty_count_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + where_clause = self._setup_where_clause( + [dataset_filter, f"({field_name} is null or trim({field_name})='')"] + ) + return f"""select count(*) + from {entity_name} {where_clause}""" + + def empty_percentage_sql( + self, field_name: str, entity_name: str, dataset_filter: Optional[str] + ) -> str: + return f"""select ({self.empty_count_sql(field_name, entity_name, dataset_filter)})/count(*) + from {entity_name} {self._setup_where_clause([dataset_filter])}""" + + def _setup_where_clause(self, filters: List[Optional[str]]) -> str: + where_clause = " and ".join(f for f in filters if f) + return f"where {where_clause}" if where_clause else "" + + def metric_sql(self, assertion: FieldMetricAssertion) -> str: + metric_sql_mapping = { + FieldMetric.UNIQUE_COUNT: self.unique_count_sql, + FieldMetric.UNIQUE_PERCENTAGE: self.unique_percentage_sql, + FieldMetric.NULL_COUNT: self.null_count_sql, + FieldMetric.NULL_PERCENTAGE: self.null_percentage_sql, + FieldMetric.MIN: self.min_sql, + FieldMetric.MAX: self.max_sql, + FieldMetric.MEAN: self.mean_sql, + FieldMetric.MEDIAN: self.median_sql, + FieldMetric.STDDEV: self.stddev_sql, + FieldMetric.NEGATIVE_COUNT: self.negative_count_sql, + FieldMetric.NEGATIVE_PERCENTAGE: self.negative_percentage_sql, + FieldMetric.ZERO_COUNT: self.zero_count_sql, + FieldMetric.ZERO_PERCENTAGE: self.zero_percentage_sql, + FieldMetric.MIN_LENGTH: self.min_length_sql, + FieldMetric.MAX_LENGTH: self.max_length_sql, + FieldMetric.EMPTY_COUNT: self.empty_count_sql, + FieldMetric.EMPTY_PERCENTAGE: self.empty_percentage_sql, + } + + entity_name = ".".join(get_entity_name(assertion)) + + return metric_sql_mapping[assertion.metric]( + assertion.field, + entity_name, + ( + assertion.filters.sql + if assertion.filters and assertion.filters.sql + else None + ), + ) diff --git a/metadata-ingestion/src/datahub/integrations/assertion/snowflake/field_values_metric_sql_generator.py b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/field_values_metric_sql_generator.py new file mode 100644 index 0000000000000..b77cc971d3a45 --- /dev/null +++ b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/field_values_metric_sql_generator.py @@ -0,0 +1,283 @@ +from functools import singledispatchmethod +from typing import List, Optional + +from datahub.api.entities.assertion.assertion_operator import ( + BetweenOperator, + ContainsOperator, + EndsWithOperator, + EqualToOperator, + GreaterThanOperator, + GreaterThanOrEqualToOperator, + InOperator, + IsFalseOperator, + IsNullOperator, + IsTrueOperator, + LessThanOperator, + LessThanOrEqualToOperator, + MatchesRegexOperator, + NotEqualToOperator, + NotInOperator, + NotNullOperator, + Operators, + StartsWithOperator, +) +from datahub.api.entities.assertion.field_assertion import ( + FieldTransform, + FieldValuesAssertion, +) +from datahub.integrations.assertion.common import get_entity_name + + +class SnowflakeFieldValuesMetricSQLGenerator: + @singledispatchmethod + def values_metric_sql( + self, + operators: Operators, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + """ + Generates SQL that would return boolean value for each table row. + 1 if FAIL and 0 if PASS. Note the unusual reversal of 1 and 0. + This is deliberate, as metric represents number of failing rows. + """ + raise ValueError(f"Unsupported values metric operator type {type(operators)} ") + + @values_metric_sql.register + def _( + self, + operators: InOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when {transformed_field} in {tuple(operators.value)} then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: NotInOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when {transformed_field} not in {tuple(operators.value)} then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: EqualToOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when {transformed_field} = {operators.value} then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: NotEqualToOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when {transformed_field} != {operators.value} then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: BetweenOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when {transformed_field} between {operators.min} and {operators.max} then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: LessThanOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when {transformed_field} < {operators.value} then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: LessThanOrEqualToOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when {transformed_field} <= {operators.value} then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: GreaterThanOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when {transformed_field} > {operators.value} then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: GreaterThanOrEqualToOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when {transformed_field} >= {operators.value} then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: IsNullOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when {transformed_field} is null then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: NotNullOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when {transformed_field} is not null then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: IsTrueOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when {transformed_field} then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: IsFalseOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when not {transformed_field} then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: ContainsOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when contains({transformed_field},'{operators.value}') then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: StartsWithOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when startswith({transformed_field},'{operators.value}') then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: EndsWithOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when endswith({transformed_field},'{operators.value}') then 0 else 1 end + from {entity_name} {where_clause}""" + + @values_metric_sql.register + def _( + self, + operators: MatchesRegexOperator, + entity_name: str, + transformed_field: str, + where_clause: str, + ) -> str: + return f"""select case when REGEXP_LIKE({transformed_field},'{operators.value}') then 0 else 1 end + from {entity_name} {where_clause}""" + + def _setup_where_clause(self, filters: List[Optional[str]]) -> str: + where_clause = " and ".join(f for f in filters if f) + return f"where {where_clause}" if where_clause else "" + + def _setup_field_transform( + self, field: str, transform: Optional[FieldTransform] + ) -> str: + if transform is None: + return field + elif transform is FieldTransform.LENGTH: + return f"length({field})" + raise ValueError(f"Unsupported transform type {transform}") + + def metric_sql(self, assertion: FieldValuesAssertion) -> str: + """ + Note that this applies negative operator in order to check whether or not + number of invalid value rows are less than configured failThreshold. + + Args: + assertion (FieldValuesAssertion): _description_ + + Returns: + str: _description_ + """ + entity_name = ".".join(get_entity_name(assertion)) + + dataset_filter = ( + assertion.filters.sql + if assertion.filters and assertion.filters.sql + else None + ) + where_clause = self._setup_where_clause( + [ + dataset_filter, + f"{assertion.field} is not null" if assertion.exclude_nulls else None, + ] + ) + transformed_field = self._setup_field_transform( + assertion.field, assertion.field_transform + ) + # this sql would return boolean value for each table row. 1 if fail and 0 if pass. + sql = self.values_metric_sql( + assertion.operator, entity_name, transformed_field, where_clause + ) + + # metric would be number of failing rows OR percentage of failing rows. + if assertion.failure_threshold.type == "count": + return f"select sum($1) as metric from ({sql})" + else: # percentage + return f"select sum($1)/count(*) as metric from ({sql})" diff --git a/metadata-ingestion/src/datahub/integrations/assertion/snowflake/metric_operator_sql_generator.py b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/metric_operator_sql_generator.py new file mode 100644 index 0000000000000..e7549d105b3f6 --- /dev/null +++ b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/metric_operator_sql_generator.py @@ -0,0 +1,68 @@ +from functools import singledispatchmethod + +from datahub.api.entities.assertion.assertion_operator import ( + BetweenOperator, + EqualToOperator, + GreaterThanOperator, + GreaterThanOrEqualToOperator, + IsFalseOperator, + IsNullOperator, + IsTrueOperator, + LessThanOperator, + LessThanOrEqualToOperator, + NotNullOperator, + Operators, +) + + +class SnowflakeMetricEvalOperatorSQLGenerator: + @singledispatchmethod + def operator_sql(self, operators: Operators, metric_sql: str) -> str: + """ + Generates Operator SQL that applies operator on `metric` + and returns a numeric boolean value 1 if PASS, 0 if FAIL + + """ + raise ValueError(f"Unsupported metric operator type {type(operators)} ") + + @operator_sql.register + def _(self, operators: EqualToOperator, metric_sql: str) -> str: + return f"select case when metric={operators.value} then 1 else 0 end from ({metric_sql})" + + @operator_sql.register + def _(self, operators: BetweenOperator, metric_sql: str) -> str: + return f"select case when metric between {operators.min} and {operators.max} then 1 else 0 end from ({metric_sql})" + + @operator_sql.register + def _(self, operators: LessThanOperator, metric_sql: str) -> str: + return f"select case when metric < {operators.value} then 1 else 0 end from ({metric_sql})" + + @operator_sql.register + def _(self, operators: LessThanOrEqualToOperator, metric_sql: str) -> str: + return f"select case when metric <= {operators.value} then 1 else 0 end from ({metric_sql})" + + @operator_sql.register + def _(self, operators: GreaterThanOperator, metric_sql: str) -> str: + return f"select case when metric > {operators.value} then 1 else 0 end from ({metric_sql})" + + @operator_sql.register + def _(self, operators: GreaterThanOrEqualToOperator, metric_sql: str) -> str: + return f"select case when metric >= {operators.value} then 1 else 0 end from ({metric_sql})" + + @operator_sql.register + def _(self, operators: NotNullOperator, metric_sql: str) -> str: + return ( + f"select case when metric is not null then 1 else 0 end from ({metric_sql})" + ) + + @operator_sql.register + def _(self, operators: IsNullOperator, metric_sql: str) -> str: + return f"select case when metric is null then 1 else 0 end from ({metric_sql})" + + @operator_sql.register + def _(self, operators: IsTrueOperator, metric_sql: str) -> str: + return f"select case when metric then 1 else 0 end from ({metric_sql})" + + @operator_sql.register + def _(self, operators: IsFalseOperator, metric_sql: str) -> str: + return f"select case when not metric then 1 else 0 end from ({metric_sql})" diff --git a/metadata-ingestion/src/datahub/integrations/assertion/snowflake/metric_sql_generator.py b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/metric_sql_generator.py new file mode 100644 index 0000000000000..5b079129e0a9c --- /dev/null +++ b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/metric_sql_generator.py @@ -0,0 +1,97 @@ +from dataclasses import dataclass +from functools import singledispatchmethod + +from datahub.api.entities.assertion.assertion import BaseEntityAssertion +from datahub.api.entities.assertion.field_assertion import ( + FieldMetricAssertion, + FieldValuesAssertion, +) +from datahub.api.entities.assertion.freshness_assertion import ( + FixedIntervalFreshnessAssertion, + FreshnessSourceType, +) +from datahub.api.entities.assertion.sql_assertion import ( + SqlMetricAssertion, + SqlMetricChangeAssertion, +) +from datahub.api.entities.assertion.volume_assertion import ( + RowCountChangeVolumeAssertion, + RowCountTotalVolumeAssertion, +) +from datahub.integrations.assertion.common import get_entity_name +from datahub.integrations.assertion.snowflake.field_metric_sql_generator import ( + SnowflakeFieldMetricSQLGenerator, +) +from datahub.integrations.assertion.snowflake.field_values_metric_sql_generator import ( + SnowflakeFieldValuesMetricSQLGenerator, +) + + +@dataclass +class SnowflakeMetricSQLGenerator: + field_metric_sql_generator: SnowflakeFieldMetricSQLGenerator + field_values_metric_sql_generator: SnowflakeFieldValuesMetricSQLGenerator + + @singledispatchmethod + def metric_sql( + self, + assertion: BaseEntityAssertion, + ) -> str: + """Generates Metric SQL that typically returns a numeric metric""" + raise ValueError(f"Unsupported assertion type {type(assertion)} ") + + @metric_sql.register + def _(self, assertion: RowCountChangeVolumeAssertion) -> str: + raise ValueError(f"Unsupported assertion type {type(assertion)} ") + + @metric_sql.register + def _(self, assertion: SqlMetricChangeAssertion) -> str: + raise ValueError(f"Unsupported assertion type {type(assertion)} ") + + @metric_sql.register + def _(self, assertion: FixedIntervalFreshnessAssertion) -> str: + entity_name = ".".join(get_entity_name(assertion)) + if assertion.filters and assertion.filters.sql: + where_clause = f"where {assertion.filters.sql}" + else: + where_clause = "" + + if ( + assertion.source_type == FreshnessSourceType.LAST_MODIFIED_COLUMN + and assertion.last_modified_field + ): + return f"""select timediff( + second, + max({assertion.last_modified_field}::TIMESTAMP_LTZ), + SNOWFLAKE.CORE.DATA_METRIC_SCHEDULED_TIME() + ) as metric from {entity_name} {where_clause}""" + else: + raise ValueError( + f"Unsupported freshness source type {assertion.source_type} " + ) + + @metric_sql.register + def _(self, assertion: RowCountTotalVolumeAssertion) -> str: + + # Can not use information schema here due to error - + # Data metric function body cannot refer to the non-deterministic function 'CURRENT_DATABASE_MAIN_METASTORE_ID'. + + entity_name = ".".join(get_entity_name(assertion)) + if assertion.filters and assertion.filters.sql: + where_clause = f"where {assertion.filters.sql}" + else: + where_clause = "" + return f"select count(*) as metric from {entity_name} {where_clause}" + + @metric_sql.register + def _(self, assertion: SqlMetricAssertion) -> str: + return f"select $1 as metric from ({assertion.statement})" + + @metric_sql.register + def _(self, assertion: FieldMetricAssertion) -> str: + sql = self.field_metric_sql_generator.metric_sql(assertion) + return f"select $1 as metric from ({sql})" + + @metric_sql.register + def _(self, assertion: FieldValuesAssertion) -> str: + return self.field_values_metric_sql_generator.metric_sql(assertion) diff --git a/metadata-ingestion/src/datahub/specific/datajob.py b/metadata-ingestion/src/datahub/specific/datajob.py index acbc1a860968b..2d944edeb3640 100644 --- a/metadata-ingestion/src/datahub/specific/datajob.py +++ b/metadata-ingestion/src/datahub/specific/datajob.py @@ -378,45 +378,29 @@ def set_output_datasets(self, outputs: List[Edge]) -> "DataJobPatchBuilder": ) return self - def add_input_dataset_field( - self, input: Union[Edge, Urn, str] - ) -> "DataJobPatchBuilder": + def add_input_dataset_field(self, input: Union[Urn, str]) -> "DataJobPatchBuilder": """ Adds an input dataset field to the DataJobPatchBuilder. Args: - input: The input dataset field, which can be an Edge object, Urn object, or a string. + input: The input dataset field, which can be an Urn object, or a string. Returns: The DataJobPatchBuilder instance. Raises: ValueError: If the input is not a Schema Field urn. - - Notes: - If `input` is an Edge object, it is used directly. If `input` is a Urn object or string, - it is converted to an Edge object and added with default audit stamps. """ - if isinstance(input, Edge): - input_urn: str = input.destinationUrn - input_edge: Edge = input - elif isinstance(input, (Urn, str)): - input_urn = str(input) - if not input_urn.startswith("urn:li:schemaField:"): - raise ValueError(f"Input {input} is not a Schema Field urn") - - input_edge = Edge( - destinationUrn=input_urn, - created=self._mint_auditstamp(), - lastModified=self._mint_auditstamp(), - ) + input_urn = str(input) + urn = Urn.create_from_string(input_urn) + if not urn.get_type() == "schemaField": + raise ValueError(f"Input {input} is not a Schema Field urn") - self._ensure_urn_type("schemaField", [input_edge], "add_input_dataset_field") self._add_patch( DataJobInputOutput.ASPECT_NAME, "add", path=f"/inputDatasetFields/{self.quote(input_urn)}", - value=input_edge, + value={}, ) return self @@ -467,44 +451,30 @@ def set_input_dataset_fields(self, inputs: List[Edge]) -> "DataJobPatchBuilder": return self def add_output_dataset_field( - self, output: Union[Edge, Urn, str] + self, output: Union[Urn, str] ) -> "DataJobPatchBuilder": """ Adds an output dataset field to the DataJobPatchBuilder. Args: - output: The output dataset field, which can be an Edge object, Urn object, or a string. + output: The output dataset field, which can be an Urn object, or a string. Returns: The DataJobPatchBuilder instance. Raises: ValueError: If the output is not a Schema Field urn. - - Notes: - If `output` is an Edge object, it is used directly. If `output` is a Urn object or string, - it is converted to an Edge object and added with default audit stamps. """ - if isinstance(output, Edge): - output_urn: str = output.destinationUrn - output_edge: Edge = output - elif isinstance(output, (Urn, str)): - output_urn = str(output) - if not output_urn.startswith("urn:li:schemaField:"): - raise ValueError(f"Input {input} is not a Schema Field urn") - - output_edge = Edge( - destinationUrn=output_urn, - created=self._mint_auditstamp(), - lastModified=self._mint_auditstamp(), - ) + output_urn = str(output) + urn = Urn.create_from_string(output_urn) + if not urn.get_type() == "schemaField": + raise ValueError(f"Input {output} is not a Schema Field urn") - self._ensure_urn_type("schemaField", [output_edge], "add_output_dataset_field") self._add_patch( DataJobInputOutput.ASPECT_NAME, "add", path=f"/outputDatasetFields/{self.quote(output_urn)}", - value=output_edge, + value={}, ) return self diff --git a/metadata-ingestion/src/datahub/specific/form.py b/metadata-ingestion/src/datahub/specific/form.py new file mode 100644 index 0000000000000..78182c202f716 --- /dev/null +++ b/metadata-ingestion/src/datahub/specific/form.py @@ -0,0 +1,146 @@ +from typing import List, Optional, Union + +from datahub.emitter.mcp_patch_builder import MetadataPatchProposal +from datahub.metadata.schema_classes import ( + FormInfoClass as FormInfo, + FormPromptClass, + KafkaAuditHeaderClass, + OwnerClass as Owner, + OwnershipTypeClass, + SystemMetadataClass, +) +from datahub.specific.ownership import OwnershipPatchHelper +from datahub.utilities.urns.urn import Urn + + +class FormPatchBuilder(MetadataPatchProposal): + def __init__( + self, + urn: str, + system_metadata: Optional[SystemMetadataClass] = None, + audit_header: Optional[KafkaAuditHeaderClass] = None, + ) -> None: + super().__init__( + urn, system_metadata=system_metadata, audit_header=audit_header + ) + self.ownership_patch_helper = OwnershipPatchHelper(self) + + def add_owner(self, owner: Owner) -> "FormPatchBuilder": + self.ownership_patch_helper.add_owner(owner) + return self + + def remove_owner( + self, owner: str, owner_type: Optional[OwnershipTypeClass] = None + ) -> "FormPatchBuilder": + """ + param: owner_type is optional + """ + self.ownership_patch_helper.remove_owner(owner, owner_type) + return self + + def set_owners(self, owners: List[Owner]) -> "FormPatchBuilder": + self.ownership_patch_helper.set_owners(owners) + return self + + def set_name(self, name: Optional[str] = None) -> "FormPatchBuilder": + if name is not None: + self._add_patch( + FormInfo.ASPECT_NAME, + "add", + path="/name", + value=name, + ) + return self + + def set_description(self, description: Optional[str] = None) -> "FormPatchBuilder": + if description is not None: + self._add_patch( + FormInfo.ASPECT_NAME, + "add", + path="/description", + value=description, + ) + return self + + def set_type(self, type: Optional[str] = None) -> "FormPatchBuilder": + if type is not None: + self._add_patch( + FormInfo.ASPECT_NAME, + "add", + path="/type", + value=type, + ) + return self + + def add_prompt(self, prompt: FormPromptClass) -> "FormPatchBuilder": + self._add_patch( + FormInfo.ASPECT_NAME, + "add", + path=f"/prompts/{self.quote(prompt.id)}", + value=prompt, + ) + return self + + def add_prompts(self, prompts: List[FormPromptClass]) -> "FormPatchBuilder": + for prompt in prompts: + self.add_prompt(prompt) + return self + + def remove_prompt(self, prompt_id: str) -> "FormPatchBuilder": + self._add_patch( + FormInfo.ASPECT_NAME, + "remove", + path=f"/prompts/{self.quote(prompt_id)}", + value=prompt_id, + ) + return self + + def remove_prompts(self, prompt_ids: List[str]) -> "FormPatchBuilder": + for prompt_id in prompt_ids: + self.remove_prompt(prompt_id) + return self + + def set_ownership_form(self, is_ownership: bool) -> "FormPatchBuilder": + self._add_patch( + FormInfo.ASPECT_NAME, + "add", + path="/actors/owners", + value=is_ownership, + ) + return self + + def add_assigned_user(self, user_urn: Union[str, Urn]) -> "FormPatchBuilder": + self._add_patch( + FormInfo.ASPECT_NAME, + "add", + path=f"/actors/users/{self.quote(str(user_urn))}", + value=user_urn, + ) + return self + + def remove_assigned_user(self, user_urn: Union[str, Urn]) -> "FormPatchBuilder": + self._add_patch( + FormInfo.ASPECT_NAME, + "remove", + path=f"/actors/users/{self.quote(str(user_urn))}", + value=user_urn, + ) + return self + + def add_assigned_group(self, group_urn: Union[str, Urn]) -> "FormPatchBuilder": + self._add_patch( + FormInfo.ASPECT_NAME, + "add", + path=f"/actors/groups/{self.quote(str(group_urn))}", + value=group_urn, + ) + return self + + def remove_assigned_group(self, group_urn: Union[str, Urn]) -> "FormPatchBuilder": + self._add_patch( + FormInfo.ASPECT_NAME, + "remove", + path=f"/actors/groups/{self.quote(str(group_urn))}", + value=group_urn, + ) + return self diff --git a/metadata-ingestion/src/datahub/specific/structured_property.py b/metadata-ingestion/src/datahub/specific/structured_property.py new file mode 100644 index 0000000000000..50f1f079c2aa7 --- /dev/null +++ b/metadata-ingestion/src/datahub/specific/structured_property.py @@ -0,0 +1,125 @@ +from typing import Dict, List, Optional, Union + +from datahub.emitter.mcp_patch_builder import MetadataPatchProposal +from datahub.metadata.schema_classes import ( + KafkaAuditHeaderClass, + PropertyValueClass, + StructuredPropertyDefinitionClass as StructuredPropertyDefinition, + SystemMetadataClass, +) +from datahub.utilities.urns.urn import Urn + + +# This patch builder is for structured property entities. Not for the aspect on assets. +class StructuredPropertyPatchBuilder(MetadataPatchProposal): + def __init__( + self, + urn: str, + system_metadata: Optional[SystemMetadataClass] = None, + audit_header: Optional[KafkaAuditHeaderClass] = None, + ) -> None: + super().__init__( + urn, system_metadata=system_metadata, audit_header=audit_header + ) + + # can only be used when creating a new structured property + def set_qualified_name( + self, qualified_name: str + ) -> "StructuredPropertyPatchBuilder": + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path="/qualifiedName", + value=qualified_name, + ) + return self + + def set_display_name( + self, display_name: Optional[str] = None + ) -> "StructuredPropertyPatchBuilder": + if display_name is not None: + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path="/displayName", + value=display_name, + ) + return self + + # can only be used when creating a new structured property + def set_value_type( + self, value_type: Union[str, Urn] + ) -> "StructuredPropertyPatchBuilder": + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path="/valueType", + value=value_type, + ) + return self + + # can only be used when creating a new structured property + def set_type_qualifier( + self, type_qualifier: Optional[Dict[str, List[str]]] = None + ) -> "StructuredPropertyPatchBuilder": + if type_qualifier is not None: + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path="/typeQualifier", + value=type_qualifier, + ) + return self + + # can only be used when creating a new structured property + def add_allowed_value( + self, allowed_value: PropertyValueClass + ) -> "StructuredPropertyPatchBuilder": + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path=f"/allowedValues/{str(allowed_value.get('value'))}", + value=allowed_value, + ) + return self + + def set_cardinality(self, cardinality: str) -> "StructuredPropertyPatchBuilder": + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path="/cardinality", + value=cardinality, + ) + return self + + def add_entity_type( + self, entity_type: Union[str, Urn] + ) -> "StructuredPropertyPatchBuilder": + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path=f"/entityTypes/{self.quote(str(entity_type))}", + value=entity_type, + ) + return self + + def set_description( + self, description: Optional[str] = None + ) -> "StructuredPropertyPatchBuilder": + if description is not None: + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path="/description", + value=description, + ) + return self + + def set_immutable(self, immutable: bool) -> "StructuredPropertyPatchBuilder": + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path="/immutable", + value=immutable, + ) + return self diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py index 530764e8320cd..677b96269fe58 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py @@ -1,6 +1,7 @@ import contextlib import dataclasses import enum +import functools import itertools import json import logging @@ -14,6 +15,7 @@ import datahub.emitter.mce_builder as builder import datahub.metadata.schema_classes as models +from datahub.configuration.time_window_config import get_time_bucket from datahub.emitter.mce_builder import get_sys_time, make_ts_millis from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.sql_parsing_builder import compute_upstream_fields @@ -30,7 +32,7 @@ SchemaFieldUrn, ) from datahub.sql_parsing.schema_resolver import SchemaResolver, SchemaResolverInterface -from datahub.sql_parsing.sql_parsing_common import QueryType +from datahub.sql_parsing.sql_parsing_common import QueryType, QueryTypeProps from datahub.sql_parsing.sqlglot_lineage import ( ColumnLineageInfo, ColumnRef, @@ -103,6 +105,7 @@ class QueryMetadata: upstreams: List[UrnStr] # this is direct upstreams, which may be temp tables column_lineage: List[ColumnLineageInfo] + column_usage: Dict[UrnStr, Set[UrnStr]] confidence_score: float used_temp_tables: bool = True @@ -128,12 +131,44 @@ class KnownQueryLineageInfo: downstream: UrnStr upstreams: List[UrnStr] column_lineage: Optional[List[ColumnLineageInfo]] = None + column_usage: Optional[Dict[UrnStr, Set[UrnStr]]] = None timestamp: Optional[datetime] = None session_id: Optional[str] = None query_type: QueryType = QueryType.UNKNOWN +@dataclasses.dataclass +class KnownLineageMapping: + upstream_urn: UrnStr + downstream_urn: UrnStr + lineage_type: str = models.DatasetLineageTypeClass.COPY + + +@dataclasses.dataclass +class PreparsedQuery: + # If not provided, we will generate one using the fast fingerprint generator. + query_id: Optional[QueryId] + + query_text: str + + upstreams: List[UrnStr] + downstream: Optional[UrnStr] = None + column_lineage: Optional[List[ColumnLineageInfo]] = None + column_usage: Optional[Dict[UrnStr, Set[UrnStr]]] = None + inferred_schema: Optional[List[models.SchemaFieldClass]] = None + confidence_score: float = 1.0 + + query_count: int = 1 + user: Optional[CorpUserUrn] = None + timestamp: Optional[datetime] = None + session_id: str = _MISSING_SESSION_ID + query_type: QueryType = QueryType.UNKNOWN + query_type_props: QueryTypeProps = dataclasses.field( + default_factory=lambda: QueryTypeProps() + ) + + @dataclasses.dataclass class SqlAggregatorReport(Report): _aggregator: "SqlParsingAggregator" @@ -160,9 +195,12 @@ class SqlAggregatorReport(Report): # SQL parsing (over all invocations). num_sql_parsed: int = 0 sql_parsing_timer: PerfTimer = dataclasses.field(default_factory=PerfTimer) + sql_fingerprinting_timer: PerfTimer = dataclasses.field(default_factory=PerfTimer) + sql_formatting_timer: PerfTimer = dataclasses.field(default_factory=PerfTimer) # Other lineage loading metrics. num_known_query_lineage: int = 0 + num_preparsed_queries: int = 0 num_known_mapping_lineage: int = 0 num_table_renames: int = 0 @@ -181,10 +219,19 @@ class SqlAggregatorReport(Report): schema_resolver_count: Optional[int] = None num_unique_query_fingerprints: Optional[int] = None num_urns_with_lineage: Optional[int] = None + num_lineage_skipped_due_to_filters: int = 0 + + # Queries. num_queries_entities_generated: int = 0 + num_queries_skipped_due_to_filters: int = 0 # Usage-related. usage_skipped_missing_timestamp: int = 0 + num_query_usage_stats_generated: int = 0 + + # Operation-related. + num_operations_generated: int = 0 + num_operations_skipped_due_to_filters: int = 0 def compute_stats(self) -> None: self.schema_resolver_count = self._aggregator._schema_resolver.schema_count() @@ -207,10 +254,13 @@ def __init__( graph: Optional[DataHubGraph] = None, generate_lineage: bool = True, generate_queries: bool = True, + generate_query_subject_fields: bool = True, generate_usage_statistics: bool = False, + generate_query_usage_statistics: bool = False, generate_operations: bool = False, usage_config: Optional[BaseUsageConfig] = None, - is_temp_table: Optional[Callable[[UrnStr], bool]] = None, + is_temp_table: Optional[Callable[[str], bool]] = None, + is_allowed_table: Optional[Callable[[str], bool]] = None, format_queries: bool = True, query_log: QueryLogSetting = _DEFAULT_QUERY_LOG_SETTING, ) -> None: @@ -220,19 +270,24 @@ def __init__( self.generate_lineage = generate_lineage self.generate_queries = generate_queries + self.generate_query_subject_fields = generate_query_subject_fields self.generate_usage_statistics = generate_usage_statistics + self.generate_query_usage_statistics = generate_query_usage_statistics self.generate_operations = generate_operations if self.generate_queries and not self.generate_lineage: raise ValueError("Queries will only be generated if lineage is enabled") self.usage_config = usage_config - if self.generate_usage_statistics and self.usage_config is None: + if ( + self.generate_usage_statistics or self.generate_query_usage_statistics + ) and self.usage_config is None: raise ValueError("Usage statistics generation requires a usage config") self.report = SqlAggregatorReport(_aggregator=self) # can be used by BQ where we have a "temp_table_dataset_prefix" - self.is_temp_table = is_temp_table + self._is_temp_table = is_temp_table + self._is_allowed_table = is_allowed_table self.format_queries = format_queries self.query_log = query_log @@ -323,6 +378,15 @@ def __init__( assert self.usage_config is not None self._usage_aggregator = UsageAggregator(config=self.usage_config) + # Query usage aggregator. + # Map of query ID -> { bucket -> count } + self._query_usage_counts: Optional[FileBackedDict[Dict[datetime, int]]] = None + if self.generate_query_usage_statistics: + self._query_usage_counts = FileBackedDict[Dict[datetime, int]]( + shared_connection=self._shared_connection, + tablename="query_usage_counts", + ) + def close(self) -> None: self._exit_stack.close() @@ -381,9 +445,44 @@ def _initialize_schema_resolver_from_graph(self, graph: DataHubGraph) -> None: def _maybe_format_query(self, query: str) -> str: if self.format_queries: - return try_format_query(query, self.platform.platform_name) + with self.report.sql_formatting_timer: + return try_format_query(query, self.platform.platform_name) return query + @functools.lru_cache(maxsize=128) + def _name_from_urn(self, urn: UrnStr) -> str: + name = DatasetUrn.from_string(urn).name + if ( + platform_instance := self._schema_resolver.platform_instance + ) and name.startswith(platform_instance): + # Remove the platform instance from the name. + name = name[len(platform_instance) + 1 :] + return name + + def is_temp_table(self, urn: UrnStr) -> bool: + if self._is_temp_table is None: + return False + return self._is_temp_table(self._name_from_urn(urn)) + + def is_allowed_table(self, urn: UrnStr) -> bool: + if self.is_temp_table(urn): + return False + if self._is_allowed_table is None: + return True + return self._is_allowed_table(self._name_from_urn(urn)) + + def add( + self, item: Union[KnownQueryLineageInfo, KnownLineageMapping, PreparsedQuery] + ) -> None: + if isinstance(item, KnownQueryLineageInfo): + self.add_known_query_lineage(item) + elif isinstance(item, KnownLineageMapping): + self.add_known_lineage_mapping(item.upstream_urn, item.downstream_urn) + elif isinstance(item, PreparsedQuery): + self.add_preparsed_query(item) + else: + raise ValueError(f"Cannot add unknown item type: {type(item)}") + def add_known_query_lineage( self, known_query_lineage: KnownQueryLineageInfo, merge_lineage: bool = False ) -> None: @@ -405,9 +504,12 @@ def add_known_query_lineage( self.report.num_known_query_lineage += 1 # Generate a fingerprint for the query. - query_fingerprint = get_query_fingerprint( - known_query_lineage.query_text, platform=self.platform.platform_name - ) + with self.report.sql_fingerprinting_timer: + query_fingerprint = get_query_fingerprint( + known_query_lineage.query_text, + platform=self.platform.platform_name, + fast=True, + ) formatted_query = self._maybe_format_query(known_query_lineage.query_text) # Register the query. @@ -422,6 +524,7 @@ def add_known_query_lineage( actor=None, upstreams=known_query_lineage.upstreams, column_lineage=known_query_lineage.column_lineage or [], + column_usage=known_query_lineage.column_usage or {}, confidence_score=1.0, ), merge_lineage=merge_lineage, @@ -474,6 +577,7 @@ def add_known_lineage_mapping( actor=None, upstreams=[upstream_urn], column_lineage=[], + column_usage={}, confidence_score=1.0, ) ) @@ -561,20 +665,69 @@ def add_observed_query( elif parsed.debug_info.column_error: self.report.num_observed_queries_column_failed += 1 + query_fingerprint = parsed.query_fingerprint + + self.add_preparsed_query( + PreparsedQuery( + query_id=query_fingerprint, + query_text=query, + query_count=usage_multiplier, + timestamp=query_timestamp, + user=user, + session_id=session_id, + query_type=parsed.query_type, + query_type_props=parsed.query_type_props, + upstreams=parsed.in_tables, + downstream=parsed.out_tables[0] if parsed.out_tables else None, + column_lineage=parsed.column_lineage, + # TODO: We need a full list of columns referenced, not just the out tables. + column_usage=compute_upstream_fields(parsed), + inferred_schema=infer_output_schema(parsed), + confidence_score=parsed.debug_info.confidence, + ), + is_known_temp_table=is_known_temp_table, + require_out_table_schema=require_out_table_schema, + session_has_temp_tables=session_has_temp_tables, + _is_internal=True, + ) + + def add_preparsed_query( + self, + parsed: PreparsedQuery, + is_known_temp_table: bool = False, + require_out_table_schema: bool = False, + session_has_temp_tables: bool = True, + _is_internal: bool = False, + ) -> None: + if not _is_internal: + self.report.num_preparsed_queries += 1 + + if parsed.timestamp: + # Sanity check - some of our usage subroutines require the timestamp to be in UTC. + # Ideally we'd actually reject missing tzinfo too, but we can tighten that later. + assert parsed.timestamp.tzinfo in {None, timezone.utc} + + query_fingerprint = parsed.query_id + if not query_fingerprint: + query_fingerprint = get_query_fingerprint( + parsed.query_text, + platform=self.platform.platform_name, + fast=True, + ) + # Format the query. - formatted_query = self._maybe_format_query(query) + formatted_query = self._maybe_format_query(parsed.query_text) # Register the query's usage. if not self._usage_aggregator: pass # usage is not enabled - elif query_timestamp is None: + elif parsed.timestamp is None: self.report.usage_skipped_missing_timestamp += 1 else: - # TODO: We need a full list of columns referenced, not just the out tables. - upstream_fields = compute_upstream_fields(parsed) - for upstream_urn in parsed.in_tables: - # If the upstream table is a temp table, don't log usage for it. - if (self.is_temp_table and self.is_temp_table(upstream_urn)) or ( + upstream_fields = parsed.column_usage or {} + for upstream_urn in parsed.upstreams: + # If the upstream table is a temp table or otherwise denied by filters, don't log usage for it. + if not self.is_allowed_table(upstream_urn) or ( require_out_table_schema and not self._schema_resolver.has_urn(upstream_urn) ): @@ -582,42 +735,49 @@ def add_observed_query( self._usage_aggregator.aggregate_event( resource=upstream_urn, - start_time=query_timestamp, + start_time=parsed.timestamp, query=formatted_query, - user=user.urn() if user else None, + user=parsed.user.urn() if parsed.user else None, fields=sorted(upstream_fields.get(upstream_urn, [])), - count=usage_multiplier, + count=parsed.query_count, ) - if not parsed.out_tables: - return - out_table = parsed.out_tables[0] - query_fingerprint = parsed.query_fingerprint - assert query_fingerprint is not None - - # Handle table renames. - is_renamed_table = False - if out_table in self._table_renames: - out_table = self._table_renames[out_table] - is_renamed_table = True + if self._query_usage_counts is not None and parsed.timestamp is not None: + assert self.usage_config is not None + bucket = get_time_bucket( + parsed.timestamp, self.usage_config.bucket_duration + ) + counts = self._query_usage_counts.for_mutation(query_fingerprint, {}) + counts[bucket] = counts.get(bucket, 0) + parsed.query_count # Register the query. self._add_to_query_map( QueryMetadata( query_id=query_fingerprint, formatted_query_string=formatted_query, - session_id=session_id, + session_id=parsed.session_id, query_type=parsed.query_type, lineage_type=models.DatasetLineageTypeClass.TRANSFORMED, - latest_timestamp=query_timestamp, - actor=user, - upstreams=parsed.in_tables, + latest_timestamp=parsed.timestamp, + actor=parsed.user, + upstreams=parsed.upstreams, column_lineage=parsed.column_lineage or [], - confidence_score=parsed.debug_info.confidence, + column_usage=parsed.column_usage or {}, + confidence_score=parsed.confidence_score, used_temp_tables=session_has_temp_tables, ) ) + if not parsed.downstream: + return + out_table = parsed.downstream + + # Handle table renames. + is_renamed_table = False + if out_table in self._table_renames: + out_table = self._table_renames[out_table] + is_renamed_table = True + # Register the query's lineage. if ( is_known_temp_table @@ -628,7 +788,7 @@ def add_observed_query( or ( not is_renamed_table and ( - (self.is_temp_table and self.is_temp_table(out_table)) + self.is_temp_table(out_table) or ( require_out_table_schema and not self._schema_resolver.has_urn(out_table) @@ -637,12 +797,11 @@ def add_observed_query( ) ): # Infer the schema of the output table and track it for later. - inferred_schema = infer_output_schema(parsed) - if inferred_schema is not None: - self._inferred_temp_schemas[query_fingerprint] = inferred_schema + if parsed.inferred_schema is not None: + self._inferred_temp_schemas[query_fingerprint] = parsed.inferred_schema # Also track the lineage for the temp table, for merging purposes later. - self._temp_lineage_map.for_mutation(session_id, {})[ + self._temp_lineage_map.for_mutation(parsed.session_id, {})[ out_table ] = query_fingerprint @@ -737,6 +896,7 @@ def _process_view_definition( actor=None, upstreams=parsed.in_tables, column_lineage=parsed.column_lineage or [], + column_usage=compute_upstream_fields(parsed), confidence_score=parsed.debug_info.confidence, ) ) @@ -818,6 +978,7 @@ def _add_to_query_map( # here just in case more schemas got registered in the interim. current.upstreams = new.upstreams current.column_lineage = new.column_lineage + current.column_usage = new.column_usage current.confidence_score = new.confidence_score else: # In the case of known query lineage, we might get things one at a time. @@ -832,13 +993,16 @@ def _add_to_query_map( self._query_map[query_fingerprint] = new def gen_metadata(self) -> Iterable[MetadataChangeProposalWrapper]: - # diff from v1 - we generate operations here, and it also - # generates MCPWs instead of workunits - yield from self._gen_lineage_mcps() + queries_generated: Set[QueryId] = set() + + yield from self._gen_lineage_mcps(queries_generated) + yield from self._gen_remaining_queries(queries_generated) yield from self._gen_usage_statistics_mcps() yield from self._gen_operation_mcps() - def _gen_lineage_mcps(self) -> Iterable[MetadataChangeProposalWrapper]: + def _gen_lineage_mcps( + self, queries_generated: Set[QueryId] + ) -> Iterable[MetadataChangeProposalWrapper]: if not self.generate_lineage: return @@ -850,7 +1014,6 @@ def _gen_lineage_mcps(self) -> Iterable[MetadataChangeProposalWrapper]: self._view_definitions.clear() # Generate lineage and queries. - queries_generated: Set[QueryId] = set() for downstream_urn in sorted(self._lineage_map): yield from self._gen_lineage_for_downstream( downstream_urn, queries_generated=queries_generated @@ -864,6 +1027,7 @@ def _query_type_precedence(cls, query_type: str) -> int: models.DatasetLineageTypeClass.TRANSFORMED, ] + # Lower value = higher precedence. idx = query_precedence.index(query_type) if idx == -1: return len(query_precedence) @@ -872,6 +1036,10 @@ def _query_type_precedence(cls, query_type: str) -> int: def _gen_lineage_for_downstream( self, downstream_urn: str, queries_generated: Set[QueryId] ) -> Iterable[MetadataChangeProposalWrapper]: + if not self.is_allowed_table(downstream_urn): + self.report.num_lineage_skipped_due_to_filters += 1 + return + query_ids = self._lineage_map[downstream_urn] queries: List[QueryMetadata] = [ self._resolve_query_with_temp_tables(self._query_map[query_id]) @@ -879,13 +1047,17 @@ def _gen_lineage_for_downstream( ] # Sort the queries by highest precedence first, then by latest timestamp. + # In case of ties, prefer queries with a known query type. # Tricky: by converting the timestamp to a number, we also can ignore the # differences between naive and aware datetimes. queries = sorted( + # Sorted is a stable sort, so in the case of total ties, we want + # to prefer the most recently added query. reversed(queries), key=lambda query: ( self._query_type_precedence(query.lineage_type), -(make_ts_millis(query.latest_timestamp) or 0), + query.query_type == QueryType.UNKNOWN, ), ) @@ -972,41 +1144,13 @@ def _gen_lineage_for_downstream( ) for query_id in required_queries: - if not self.can_generate_query(query_id): - continue - # Avoid generating the same query twice. if query_id in queries_generated: continue queries_generated.add(query_id) - self.report.num_queries_entities_generated += 1 query = queries_map[query_id] - yield from MetadataChangeProposalWrapper.construct_many( - entityUrn=self._query_urn(query_id), - aspects=[ - models.QueryPropertiesClass( - statement=models.QueryStatementClass( - value=query.formatted_query_string, - language=models.QueryLanguageClass.SQL, - ), - source=models.QuerySourceClass.SYSTEM, - created=query.make_created_audit_stamp(), - lastModified=query.make_last_modified_audit_stamp(), - ), - models.QuerySubjectsClass( - subjects=[ - models.QuerySubjectClass(entity=dataset_urn) - for dataset_urn in itertools.chain( - [downstream_urn], query.upstreams - ) - ] - ), - models.DataPlatformInstanceClass( - platform=self.platform.urn(), - ), - ], - ) + yield from self._gen_query(query, downstream_urn) @classmethod def _query_urn(cls, query_id: QueryId) -> str: @@ -1032,9 +1176,121 @@ def _is_known_lineage_query_id(cls, query_id: QueryId) -> bool: # never conflict with a real query fingerprint. return query_id.startswith("known_") + def _gen_remaining_queries( + self, queries_generated: Set[QueryId] + ) -> Iterable[MetadataChangeProposalWrapper]: + if not self.generate_queries or not self.generate_query_usage_statistics: + return + + assert self._query_usage_counts is not None + for query_id in self._query_usage_counts: + if query_id in queries_generated: + continue + queries_generated.add(query_id) + + yield from self._gen_query(self._query_map[query_id]) + def can_generate_query(self, query_id: QueryId) -> bool: return self.generate_queries and not self._is_known_lineage_query_id(query_id) + def _gen_query( + self, query: QueryMetadata, downstream_urn: Optional[str] = None + ) -> Iterable[MetadataChangeProposalWrapper]: + query_id = query.query_id + if not self.can_generate_query(query_id): + return + + # If a query doesn't involve any allowed tables, skip it. + if downstream_urn is None and not any( + self.is_allowed_table(urn) for urn in query.upstreams + ): + self.report.num_queries_skipped_due_to_filters += 1 + return + + query_subject_urns = OrderedSet[UrnStr]() + for upstream in query.upstreams: + query_subject_urns.add(upstream) + if self.generate_query_subject_fields: + for column in query.column_usage.get(upstream, []): + query_subject_urns.add( + builder.make_schema_field_urn(upstream, column) + ) + if downstream_urn: + query_subject_urns.add(downstream_urn) + if self.generate_query_subject_fields: + for column_lineage in query.column_lineage: + query_subject_urns.add( + builder.make_schema_field_urn( + downstream_urn, column_lineage.downstream.column + ) + ) + + yield from MetadataChangeProposalWrapper.construct_many( + entityUrn=self._query_urn(query_id), + aspects=[ + models.QueryPropertiesClass( + statement=models.QueryStatementClass( + value=query.formatted_query_string, + language=models.QueryLanguageClass.SQL, + ), + source=models.QuerySourceClass.SYSTEM, + created=query.make_created_audit_stamp(), + lastModified=query.make_last_modified_audit_stamp(), + ), + models.QuerySubjectsClass( + subjects=[ + models.QuerySubjectClass(entity=urn) + for urn in query_subject_urns + ] + ), + models.DataPlatformInstanceClass( + platform=self.platform.urn(), + ), + ], + ) + self.report.num_queries_entities_generated += 1 + + if self._query_usage_counts is not None: + assert self.usage_config is not None + + # This is slightly lossy, since we only store one unique + # user per query instead of tracking all of them. + # We also lose information because we don't keep track + # of users / lastExecutedAt timestamps per bucket. + user = query.actor + + query_counter = self._query_usage_counts.get(query_id) + if not query_counter: + return + for bucket in self.usage_config.buckets(): + count = query_counter.get(bucket) + if not count: + continue + + yield MetadataChangeProposalWrapper( + entityUrn=self._query_urn(query_id), + aspect=models.QueryUsageStatisticsClass( + timestampMillis=make_ts_millis(bucket), + eventGranularity=models.TimeWindowSizeClass( + unit=self.usage_config.bucket_duration, multiple=1 + ), + queryCount=count, + uniqueUserCount=1, + userCounts=( + [ + models.DatasetUserUsageCountsClass( + user=user.urn(), + count=count, + ) + ] + if user + else None + ), + ), + ) + + self.report.num_query_usage_stats_generated += 1 + def _resolve_query_with_temp_tables( self, base_query: QueryMetadata, @@ -1199,6 +1455,11 @@ def _gen_operation_for_downstream( # We don't generate operations for SELECTs. return + if not self.is_allowed_table(downstream_urn): + self.report.num_operations_skipped_due_to_filters += 1 + return + + self.report.num_operations_generated += 1 aspect = models.OperationClass( timestampMillis=make_ts_millis(datetime.now(tz=timezone.utc)), operationType=operation_type, diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py index ddab26b28ec4f..56e4c806eb0c3 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py @@ -1,6 +1,7 @@ import functools import hashlib import logging +import re from typing import Dict, Iterable, Optional, Tuple, Union import sqlglot @@ -65,7 +66,7 @@ def _parse_statement( sql: sqlglot.exp.ExpOrStr, dialect: sqlglot.Dialect ) -> sqlglot.Expression: statement: sqlglot.Expression = sqlglot.maybe_parse( - sql, dialect=dialect, error_level=sqlglot.ErrorLevel.RAISE + sql, dialect=dialect, error_level=sqlglot.ErrorLevel.IMMEDIATE ) return statement @@ -109,6 +110,80 @@ def _expression_to_string( return expression.sql(dialect=get_dialect(platform)) +_BASIC_NORMALIZATION_RULES = { + # Remove /* */ comments. + re.compile(r"/\*.*?\*/", re.DOTALL): "", + # Remove -- comments. + re.compile(r"--.*$"): "", + # Replace all runs of whitespace with a single space. + re.compile(r"\s+"): " ", + # Remove leading and trailing whitespace and trailing semicolons. + re.compile(r"^\s+|[\s;]+$"): "", + # Replace anything that looks like a number with a placeholder. + re.compile(r"\b\d+\b"): "?", + # Replace anything that looks like a string with a placeholder. + re.compile(r"'[^']*'"): "?", + # Replace sequences of IN/VALUES with a single placeholder. + re.compile(r"\b(IN|VALUES)\s*\(\?(?:, \?)*\)", re.IGNORECASE): r"\1 (?)", + # Normalize parenthesis spacing. + re.compile(r"\( "): "(", + re.compile(r" \)"): ")", +} +_TABLE_NAME_NORMALIZATION_RULES = { + # Replace UUID-like strings with a placeholder (both - and _ variants). + re.compile( + r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + re.IGNORECASE, + ): "00000000-0000-0000-0000-000000000000", + re.compile( + r"[0-9a-f]{8}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{12}", + re.IGNORECASE, + ): "00000000_0000_0000_0000_000000000000", + # GE temporary table names (prefix + 8 digits of a UUIDv4) + re.compile( + r"\b(ge_tmp_|ge_temp_|gx_temp_)[0-9a-f]{8}\b", re.IGNORECASE + ): r"\1abcdefgh", + # Date-suffixed table names (e.g. _20210101) + re.compile(r"\b(\w+)(19|20)\d{4}\b"): r"\1YYYYMM", + re.compile(r"\b(\w+)(19|20)\d{6}\b"): r"\1YYYYMMDD", + re.compile(r"\b(\w+)(19|20)\d{8}\b"): r"\1YYYYMMDDHH", + re.compile(r"\b(\w+)(19|20)\d{10}\b"): r"\1YYYYMMDDHHMM", +} + + +def generalize_query_fast( + expression: sqlglot.exp.ExpOrStr, + dialect: DialectOrStr, + change_table_names: bool = False, +) -> str: + """Variant of `generalize_query` that only does basic normalization. + + Args: + expression: The SQL query to generalize. + dialect: The SQL dialect to use. + change_table_names: If True, replace table names with placeholders. Note + that this should only be used for query filtering purposes, as it + violates the general assumption that the queries with the same fingerprint + have the same lineage/usage/etc. + + Returns: + The generalized SQL query. + """ + + if isinstance(expression, sqlglot.exp.Expression): + expression = expression.sql(dialect=get_dialect(dialect)) + query_text = expression + + REGEX_REPLACEMENTS = { + **_BASIC_NORMALIZATION_RULES, + **(_TABLE_NAME_NORMALIZATION_RULES if change_table_names else {}), + } + + for pattern, replacement in REGEX_REPLACEMENTS.items(): + query_text = pattern.sub(replacement, query_text) + return query_text + + def generalize_query(expression: sqlglot.exp.ExpOrStr, dialect: DialectOrStr) -> str: """ Generalize/normalize a SQL query. @@ -172,11 +247,14 @@ def generate_hash(text: str) -> str: def get_query_fingerprint_debug( - expression: sqlglot.exp.ExpOrStr, platform: DialectOrStr + expression: sqlglot.exp.ExpOrStr, platform: DialectOrStr, fast: bool = False ) -> Tuple[str, Optional[str]]: try: - dialect = get_dialect(platform) - expression_sql = generalize_query(expression, dialect=dialect) + if not fast: + dialect = get_dialect(platform) + expression_sql = generalize_query(expression, dialect=dialect) + else: + expression_sql = generalize_query_fast(expression, dialect=platform) except (ValueError, sqlglot.errors.SqlglotError) as e: if not isinstance(expression, str): raise @@ -193,7 +271,7 @@ def get_query_fingerprint_debug( def get_query_fingerprint( - expression: sqlglot.exp.ExpOrStr, platform: DialectOrStr + expression: sqlglot.exp.ExpOrStr, platform: DialectOrStr, fast: bool = False ) -> str: """Get a fingerprint for a SQL query. @@ -215,7 +293,7 @@ def get_query_fingerprint( The fingerprint for the SQL query. """ - return get_query_fingerprint_debug(expression, platform)[0] + return get_query_fingerprint_debug(expression, platform, fast=fast)[0] @functools.lru_cache(maxsize=FORMAT_QUERY_CACHE_SIZE) @@ -237,8 +315,8 @@ def try_format_query( try: dialect = get_dialect(platform) - expression = parse_statement(expression, dialect=dialect) - return expression.sql(dialect=dialect, pretty=True) + parsed_expression = parse_statement(expression, dialect=dialect) + return parsed_expression.sql(dialect=dialect, pretty=True) except Exception as e: if raises: raise diff --git a/metadata-ingestion/src/datahub/utilities/advanced_thread_executor.py b/metadata-ingestion/src/datahub/utilities/advanced_thread_executor.py deleted file mode 100644 index 1e241a9a49e97..0000000000000 --- a/metadata-ingestion/src/datahub/utilities/advanced_thread_executor.py +++ /dev/null @@ -1,231 +0,0 @@ -from __future__ import annotations - -import collections -import concurrent.futures -import logging -import time -from concurrent.futures import Future, ThreadPoolExecutor -from threading import BoundedSemaphore -from typing import ( - Any, - Callable, - Deque, - Dict, - Iterable, - Iterator, - Optional, - Set, - Tuple, - TypeVar, -) - -from datahub.ingestion.api.closeable import Closeable - -logger = logging.getLogger(__name__) -_R = TypeVar("_R") -_PARTITION_EXECUTOR_FLUSH_SLEEP_INTERVAL = 0.05 - - -class PartitionExecutor(Closeable): - def __init__(self, max_workers: int, max_pending: int) -> None: - """A thread pool executor with partitioning and a pending request bound. - - It works similarly to a ThreadPoolExecutor, with the following changes: - - At most one request per partition key will be executing at a time. - - If the number of pending requests exceeds the threshold, the submit() call - will block until the number of pending requests drops below the threshold. - - Due to the interaction between max_workers and max_pending, it is possible - for execution to effectively be serialized when there's a large influx of - requests with the same key. This can be mitigated by setting a reasonably - large max_pending value. - - Args: - max_workers: The maximum number of threads to use for executing requests. - max_pending: The maximum number of pending (e.g. non-executing) requests to allow. - """ - self.max_workers = max_workers - self.max_pending = max_pending - - self._executor = ThreadPoolExecutor(max_workers=max_workers) - - # Each pending or executing request will acquire a permit from this semaphore. - self._semaphore = BoundedSemaphore(max_pending + max_workers) - - # A key existing in this dict means that there is a submitted request for that key. - # Any entries in the key's value e.g. the deque are requests that are waiting - # to be submitted once the current request for that key completes. - self._pending_by_key: Dict[ - str, Deque[Tuple[Callable, tuple, dict, Optional[Callable[[Future], None]]]] - ] = {} - - def submit( - self, - key: str, - fn: Callable[..., _R], - *args: Any, - # Ideally, we would've used ParamSpec to annotate this method. However, - # due to the limitations of PEP 612, we can't add a keyword argument here. - # See https://peps.python.org/pep-0612/#concatenating-keyword-parameters - # As such, we're using Any here, and won't validate the args to this method. - # We might be able to work around it by moving the done_callback arg to be before - # the *args, but that would mean making done_callback a required arg instead of - # optional as it is now. - done_callback: Optional[Callable[[Future], None]] = None, - **kwargs: Any, - ) -> None: - """See concurrent.futures.Executor#submit""" - - self._semaphore.acquire() - - if key in self._pending_by_key: - self._pending_by_key[key].append((fn, args, kwargs, done_callback)) - - else: - self._pending_by_key[key] = collections.deque() - self._submit_nowait(key, fn, args, kwargs, done_callback=done_callback) - - def _submit_nowait( - self, - key: str, - fn: Callable[..., _R], - args: tuple, - kwargs: dict, - done_callback: Optional[Callable[[Future], None]], - ) -> Future: - future = self._executor.submit(fn, *args, **kwargs) - - def _system_done_callback(future: Future) -> None: - self._semaphore.release() - - # If there is another pending request for this key, submit it now. - # The key must exist in the map. - if self._pending_by_key[key]: - fn, args, kwargs, user_done_callback = self._pending_by_key[ - key - ].popleft() - - try: - self._submit_nowait(key, fn, args, kwargs, user_done_callback) - except RuntimeError as e: - if self._executor._shutdown: - # If we're in shutdown mode, then we can't submit any more requests. - # That means we'll need to drop requests on the floor, which is to - # be expected in shutdown mode. - # The only reason we'd normally be in shutdown here is during - # Python exit (e.g. KeyboardInterrupt), so this is reasonable. - logger.debug("Dropping request due to shutdown") - else: - raise e - - else: - # If there are no pending requests for this key, mark the key - # as no longer in progress. - del self._pending_by_key[key] - - if done_callback: - future.add_done_callback(done_callback) - future.add_done_callback(_system_done_callback) - return future - - def flush(self) -> None: - """Wait for all pending requests to complete.""" - - # Acquire all the semaphore permits so that no more requests can be submitted. - for _i in range(self.max_pending): - self._semaphore.acquire() - - # Now, wait for all the pending requests to complete. - while len(self._pending_by_key) > 0: - # TODO: There should be a better way to wait for all executor threads to be idle. - # One option would be to just shutdown the existing executor and create a new one. - time.sleep(_PARTITION_EXECUTOR_FLUSH_SLEEP_INTERVAL) - - # Now allow new requests to be submitted. - # TODO: With Python 3.9, release() can take a count argument. - for _i in range(self.max_pending): - self._semaphore.release() - - def shutdown(self) -> None: - """See concurrent.futures.Executor#shutdown. Behaves as if wait=True.""" - - self.flush() - assert len(self._pending_by_key) == 0 - - # Technically, the wait=True here is redundant, since all the threads should - # be idle now. - self._executor.shutdown(wait=True) - - def close(self) -> None: - self.shutdown() - - -class BackpressureAwareExecutor: - # This couldn't be a real executor because the semantics of submit wouldn't really make sense. - # In this variant, if we blocked on submit, then we would also be blocking the thread that - # we expect to be consuming the results. As such, I made it accept the full list of args - # up front, and that way the consumer can read results at its own pace. - - @classmethod - def map( - cls, - fn: Callable[..., _R], - args_list: Iterable[Tuple[Any, ...]], - max_workers: int, - max_pending: Optional[int] = None, - ) -> Iterator[Future[_R]]: - """Similar to concurrent.futures.ThreadPoolExecutor#map, except that it won't run ahead of the consumer. - - The main benefit is that the ThreadPoolExecutor isn't stuck holding a ton of result - objects in memory if the consumer is slow. Instead, the consumer can read the results - at its own pace and the executor threads will idle if they need to. - - Args: - fn: The function to apply to each input. - args_list: The list of inputs. In contrast to the builtin map, this is a list - of tuples, where each tuple is the arguments to fn. - max_workers: The maximum number of threads to use. - max_pending: The maximum number of pending results to keep in memory. - If not set, it will be set to 2*max_workers. - - Returns: - An iterable of futures. - - This differs from a traditional map because it returns futures - instead of the actual results, so that the caller is required - to handle exceptions. - - Additionally, it does not maintain the order of the arguments. - If you want to know which result corresponds to which input, - the mapped function should return some form of an identifier. - """ - - if max_pending is None: - max_pending = 2 * max_workers - assert max_pending >= max_workers - - pending_futures: Set[Future] = set() - - with ThreadPoolExecutor(max_workers=max_workers) as executor: - for args in args_list: - # If the pending list is full, wait until one is done. - if len(pending_futures) >= max_pending: - (done, _) = concurrent.futures.wait( - pending_futures, return_when=concurrent.futures.FIRST_COMPLETED - ) - for future in done: - pending_futures.remove(future) - - # We don't want to call result() here because we want the caller - # to handle exceptions/cancellation. - yield future - - # Now that there's space in the pending list, enqueue the next task. - pending_futures.add(executor.submit(fn, *args)) - - # Wait for all the remaining tasks to complete. - for future in concurrent.futures.as_completed(pending_futures): - pending_futures.remove(future) - yield future - - assert not pending_futures diff --git a/metadata-ingestion/src/datahub/utilities/backpressure_aware_executor.py b/metadata-ingestion/src/datahub/utilities/backpressure_aware_executor.py new file mode 100644 index 0000000000000..988bd91c4a642 --- /dev/null +++ b/metadata-ingestion/src/datahub/utilities/backpressure_aware_executor.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import concurrent.futures +from concurrent.futures import Future, ThreadPoolExecutor +from typing import Any, Callable, Iterable, Iterator, Optional, Set, Tuple, TypeVar + +_R = TypeVar("_R") + + +class BackpressureAwareExecutor: + # This couldn't be a real executor because the semantics of submit wouldn't really make sense. + # In this variant, if we blocked on submit, then we would also be blocking the thread that + # we expect to be consuming the results. As such, I made it accept the full list of args + # up front, and that way the consumer can read results at its own pace. + + @classmethod + def map( + cls, + fn: Callable[..., _R], + args_list: Iterable[Tuple[Any, ...]], + max_workers: int, + max_pending: Optional[int] = None, + ) -> Iterator[Future[_R]]: + """Similar to concurrent.futures.ThreadPoolExecutor#map, except that it won't run ahead of the consumer. + + The main benefit is that the ThreadPoolExecutor isn't stuck holding a ton of result + objects in memory if the consumer is slow. Instead, the consumer can read the results + at its own pace and the executor threads will idle if they need to. + + Args: + fn: The function to apply to each input. + args_list: The list of inputs. In contrast to the builtin map, this is a list + of tuples, where each tuple is the arguments to fn. + max_workers: The maximum number of threads to use. + max_pending: The maximum number of pending results to keep in memory. + If not set, it will be set to 2*max_workers. + + Returns: + An iterable of futures. + + This differs from a traditional map because it returns futures + instead of the actual results, so that the caller is required + to handle exceptions. + + Additionally, it does not maintain the order of the arguments. + If you want to know which result corresponds to which input, + the mapped function should return some form of an identifier. + """ + + if max_pending is None: + max_pending = 2 * max_workers + assert max_pending >= max_workers + + pending_futures: Set[Future] = set() + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + for args in args_list: + # If the pending list is full, wait until one is done. + if len(pending_futures) >= max_pending: + (done, _) = concurrent.futures.wait( + pending_futures, return_when=concurrent.futures.FIRST_COMPLETED + ) + for future in done: + pending_futures.remove(future) + + # We don't want to call result() here because we want the caller + # to handle exceptions/cancellation. + yield future + + # Now that there's space in the pending list, enqueue the next task. + pending_futures.add(executor.submit(fn, *args)) + + # Wait for all the remaining tasks to complete. + for future in concurrent.futures.as_completed(pending_futures): + pending_futures.remove(future) + yield future + + assert not pending_futures diff --git a/metadata-ingestion/src/datahub/utilities/logging_manager.py b/metadata-ingestion/src/datahub/utilities/logging_manager.py index 6bfbb9d373396..64383745eb2d1 100644 --- a/metadata-ingestion/src/datahub/utilities/logging_manager.py +++ b/metadata-ingestion/src/datahub/utilities/logging_manager.py @@ -149,9 +149,9 @@ def filter(self, record: logging.LogRecord) -> bool: return record.levelno >= logging.INFO else: if self.debug: - return record.levelno >= logging.WARNING - else: return record.levelno >= logging.INFO + else: + return record.levelno >= logging.WARNING class _LogBuffer: @@ -274,6 +274,7 @@ def configure_logging(debug: bool, log_file: Optional[str] = None) -> Iterator[N # Reduce logging from some particularly chatty libraries. logging.getLogger("urllib3").setLevel(logging.ERROR) +logging.getLogger("urllib3.util.retry").setLevel(logging.WARNING) logging.getLogger("snowflake").setLevel(level=logging.WARNING) # logging.getLogger("botocore").setLevel(logging.INFO) # logging.getLogger("google").setLevel(logging.INFO) diff --git a/metadata-ingestion/src/datahub/utilities/lossy_collections.py b/metadata-ingestion/src/datahub/utilities/lossy_collections.py index 6f5f4bda369a0..f71aad51ab0b6 100644 --- a/metadata-ingestion/src/datahub/utilities/lossy_collections.py +++ b/metadata-ingestion/src/datahub/utilities/lossy_collections.py @@ -58,11 +58,19 @@ def __get_pydantic_core_schema__(cls, source_type, handler): # type: ignore return core_schema.no_info_after_validator_function(cls, handler(list)) def as_obj(self) -> List[Union[T, str]]: - base_list: List[Union[T, str]] = list(self.__iter__()) + from datahub.ingestion.api.report import Report + + base_list: List[Union[T, str]] = [ + Report.to_pure_python_obj(value) for value in list(self.__iter__()) + ] if self.sampled: base_list.append(f"... sampled of {self.total_elements} total elements") return base_list + def set_total(self, total: int) -> None: + self.total_elements = total + self.sampled = self.total_elements > self.max_elements + class LossySet(Set[T], Generic[T]): """A set that only preserves a sample of elements in a set. Currently this is a very simple greedy sampling set""" @@ -145,9 +153,13 @@ def as_obj(self) -> Dict[Union[_KT, str], Union[_VT, str]]: if self.sampled: base_dict[ "sampled" - ] = f"{len(self.keys())} sampled of at most {self.max_elements + self._overflow} entries." + ] = f"{len(self.keys())} sampled of at most {self.total_key_count()} entries." return base_dict + def total_key_count(self) -> int: + """Returns the total number of keys that have been added to this dictionary.""" + return super().__len__() + self._overflow + def dropped_keys_count(self) -> int: """Returns the number of keys that have been dropped from this dictionary.""" return self._overflow diff --git a/metadata-ingestion/src/datahub/utilities/partition_executor.py b/metadata-ingestion/src/datahub/utilities/partition_executor.py new file mode 100644 index 0000000000000..05e81da47285d --- /dev/null +++ b/metadata-ingestion/src/datahub/utilities/partition_executor.py @@ -0,0 +1,404 @@ +from __future__ import annotations + +import collections +import functools +import logging +import queue +import threading +import time +from concurrent.futures import Future, ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +from threading import BoundedSemaphore +from typing import ( + Any, + Callable, + Deque, + Dict, + List, + NamedTuple, + Optional, + Set, + Tuple, + TypeVar, +) + +from datahub.ingestion.api.closeable import Closeable + +logger = logging.getLogger(__name__) +_R = TypeVar("_R") +_Args = TypeVar("_Args", bound=tuple) +_PARTITION_EXECUTOR_FLUSH_SLEEP_INTERVAL = 0.05 +_DEFAULT_BATCHER_MIN_PROCESS_INTERVAL = timedelta(seconds=30) + + +class PartitionExecutor(Closeable): + def __init__(self, max_workers: int, max_pending: int) -> None: + """A thread pool executor with partitioning and a pending request bound. + + It works similarly to a ThreadPoolExecutor, with the following changes: + - At most one request per partition key will be executing at a time. + - If the number of pending requests exceeds the threshold, the submit() call + will block until the number of pending requests drops below the threshold. + + Due to the interaction between max_workers and max_pending, it is possible + for execution to effectively be serialized when there's a large influx of + requests with the same key. This can be mitigated by setting a reasonably + large max_pending value. + + Args: + max_workers: The maximum number of threads to use for executing requests. + max_pending: The maximum number of pending (e.g. non-executing) requests to allow. + """ + self.max_workers = max_workers + self.max_pending = max_pending + + self._executor = ThreadPoolExecutor(max_workers=max_workers) + + # Each pending or executing request will acquire a permit from this semaphore. + self._semaphore = BoundedSemaphore(max_pending + max_workers) + + # A key existing in this dict means that there is a submitted request for that key. + # Any entries in the key's value e.g. the deque are requests that are waiting + # to be submitted once the current request for that key completes. + self._pending_by_key: Dict[ + str, Deque[Tuple[Callable, tuple, dict, Optional[Callable[[Future], None]]]] + ] = {} + + def submit( + self, + key: str, + fn: Callable[..., _R], + *args: Any, + # Ideally, we would've used ParamSpec to annotate this method. However, + # due to the limitations of PEP 612, we can't add a keyword argument here. + # See https://peps.python.org/pep-0612/#concatenating-keyword-parameters + # As such, we're using Any here, and won't validate the args to this method. + # We might be able to work around it by moving the done_callback arg to be before + # the *args, but that would mean making done_callback a required arg instead of + # optional as it is now. + done_callback: Optional[Callable[[Future], None]] = None, + **kwargs: Any, + ) -> None: + """See concurrent.futures.Executor#submit""" + + self._semaphore.acquire() + + if key in self._pending_by_key: + self._pending_by_key[key].append((fn, args, kwargs, done_callback)) + + else: + self._pending_by_key[key] = collections.deque() + self._submit_nowait(key, fn, args, kwargs, done_callback=done_callback) + + def _submit_nowait( + self, + key: str, + fn: Callable[..., _R], + args: tuple, + kwargs: dict, + done_callback: Optional[Callable[[Future], None]], + ) -> Future: + future = self._executor.submit(fn, *args, **kwargs) + + def _system_done_callback(future: Future) -> None: + self._semaphore.release() + + # If there is another pending request for this key, submit it now. + # The key must exist in the map. + if self._pending_by_key[key]: + fn, args, kwargs, user_done_callback = self._pending_by_key[ + key + ].popleft() + + try: + self._submit_nowait(key, fn, args, kwargs, user_done_callback) + except RuntimeError as e: + if self._executor._shutdown: + # If we're in shutdown mode, then we can't submit any more requests. + # That means we'll need to drop requests on the floor, which is to + # be expected in shutdown mode. + # The only reason we'd normally be in shutdown here is during + # Python exit (e.g. KeyboardInterrupt), so this is reasonable. + logger.debug("Dropping request due to shutdown") + else: + raise e + + else: + # If there are no pending requests for this key, mark the key + # as no longer in progress. + del self._pending_by_key[key] + + if done_callback: + future.add_done_callback(done_callback) + future.add_done_callback(_system_done_callback) + return future + + def flush(self) -> None: + """Wait for all pending requests to complete.""" + + # Acquire all the semaphore permits so that no more requests can be submitted. + for _i in range(self.max_pending): + self._semaphore.acquire() + + # Now, wait for all the pending requests to complete. + while len(self._pending_by_key) > 0: + # TODO: There should be a better way to wait for all executor threads to be idle. + # One option would be to just shutdown the existing executor and create a new one. + time.sleep(_PARTITION_EXECUTOR_FLUSH_SLEEP_INTERVAL) + + # Now allow new requests to be submitted. + # TODO: With Python 3.9, release() can take a count argument. + for _i in range(self.max_pending): + self._semaphore.release() + + def shutdown(self) -> None: + """See concurrent.futures.Executor#shutdown. Behaves as if wait=True.""" + + self.flush() + assert len(self._pending_by_key) == 0 + + self._executor.shutdown(wait=True) + + def close(self) -> None: + self.shutdown() + + +class _BatchPartitionWorkItem(NamedTuple): + key: str + args: tuple + done_callback: Optional[Callable[[Future], None]] + + +def _now() -> datetime: + return datetime.now(tz=timezone.utc) + + +class BatchPartitionExecutor(Closeable): + def __init__( + self, + max_workers: int, + max_pending: int, + # Due to limitations of Python's typing, we can't express the type of the list + # effectively. Ideally we'd use ParamSpec here, but that's not allowed in a + # class context like this. + process_batch: Callable[[List], None], + max_per_batch: int = 100, + min_process_interval: timedelta = _DEFAULT_BATCHER_MIN_PROCESS_INTERVAL, + ) -> None: + """Similar to PartitionExecutor, but with batching. + + This takes in the stream of requests, automatically segments them into partition-aware + batches, and schedules them across a pool of worker threads. + + It maintains the invariant that multiple requests with the same key will not be in + flight concurrently, except when part of the same batch. Requests for a given key + will also be executed in the order they were submitted. + + Unlike the PartitionExecutor, this does not support return values or kwargs. + + Args: + max_workers: The maximum number of threads to use for executing requests. + max_pending: The maximum number of pending (e.g. non-executing) requests to allow. + max_per_batch: The maximum number of requests to include in a batch. + min_process_interval: When requests are coming in slowly, we will wait at least this long + before submitting a non-full batch. + process_batch: A function that takes in a list of argument tuples. + """ + self.max_workers = max_workers + self.max_pending = max_pending + self.max_per_batch = max_per_batch + self.process_batch = process_batch + self.min_process_interval = min_process_interval + assert self.max_workers > 1 + + # We add one here to account for the clearinghouse worker thread. + self._executor = ThreadPoolExecutor(max_workers=max_workers + 1) + self._clearinghouse_started = False + + self._pending_count = BoundedSemaphore(max_pending) + self._pending: "queue.Queue[Optional[_BatchPartitionWorkItem]]" = queue.Queue( + maxsize=max_pending + ) + + # If this is true, that means shutdown() has been called and self._pending is empty. + self._queue_empty_for_shutdown = False + + def _clearinghouse_worker(self) -> None: # noqa: C901 + # This worker will pull items off the queue, and submit them into the executor + # in batches. Only this worker will submit process commands to the executor thread pool. + + # The lock protects the function's internal state. + clearinghouse_state_lock = threading.Lock() + workers_available = self.max_workers + keys_in_flight: Set[str] = set() + keys_no_longer_in_flight: Set[str] = set() + pending_key_completion: List[_BatchPartitionWorkItem] = [] + + last_submit_time = _now() + + def _handle_batch_completion( + batch: List[_BatchPartitionWorkItem], future: Future + ) -> None: + with clearinghouse_state_lock: + for item in batch: + keys_no_longer_in_flight.add(item.key) + self._pending_count.release() + + # Separate from the above loop to avoid holding the lock while calling the callbacks. + for item in batch: + if item.done_callback: + item.done_callback(future) + + def _find_ready_items() -> List[_BatchPartitionWorkItem]: + with clearinghouse_state_lock: + # First, update the keys in flight. + for key in keys_no_longer_in_flight: + keys_in_flight.remove(key) + keys_no_longer_in_flight.clear() + + # Then, update the pending key completion and build the ready list. + pending = pending_key_completion.copy() + pending_key_completion.clear() + + ready: List[_BatchPartitionWorkItem] = [] + for item in pending: + if ( + len(ready) < self.max_per_batch + and item.key not in keys_in_flight + ): + ready.append(item) + else: + pending_key_completion.append(item) + + return ready + + def _build_batch() -> List[_BatchPartitionWorkItem]: + next_batch = _find_ready_items() + + while ( + not self._queue_empty_for_shutdown + and len(next_batch) < self.max_per_batch + ): + blocking = True + if ( + next_batch + and _now() - last_submit_time > self.min_process_interval + and workers_available > 0 + ): + # If we're past the submit deadline, pull from the queue + # in a non-blocking way, and submit the batch once the queue + # is empty. + blocking = False + + try: + next_item: Optional[_BatchPartitionWorkItem] = self._pending.get( + block=blocking, + timeout=self.min_process_interval.total_seconds(), + ) + if next_item is None: + self._queue_empty_for_shutdown = True + break + + with clearinghouse_state_lock: + if next_item.key in keys_in_flight: + pending_key_completion.append(next_item) + else: + next_batch.append(next_item) + except queue.Empty: + if not blocking: + break + + return next_batch + + def _submit_batch(next_batch: List[_BatchPartitionWorkItem]) -> None: + with clearinghouse_state_lock: + for item in next_batch: + keys_in_flight.add(item.key) + + nonlocal workers_available + workers_available -= 1 + + nonlocal last_submit_time + last_submit_time = _now() + + future = self._executor.submit( + self.process_batch, [item.args for item in next_batch] + ) + future.add_done_callback( + functools.partial(_handle_batch_completion, next_batch) + ) + + try: + # Normal operation - submit batches as they become available. + while not self._queue_empty_for_shutdown: + next_batch = _build_batch() + if next_batch: + _submit_batch(next_batch) + + # Shutdown time. + # Invariant - at this point, we know self._pending is empty. + # We just need to wait for the in-flight items to complete, + # and submit any currently pending items once possible. + while pending_key_completion: + next_batch = _build_batch() + if next_batch: + _submit_batch(next_batch) + time.sleep(_PARTITION_EXECUTOR_FLUSH_SLEEP_INTERVAL) + + # At this point, there are no more things to submit. + # We could wait for the in-flight items to complete, + # but the executor will take care of waiting for them to complete. + except Exception as e: + # This represents a fatal error that makes the entire executor defunct. + logger.exception( + "Threaded executor's clearinghouse worker failed.", exc_info=e + ) + finally: + self._clearinghouse_started = False + + def _ensure_clearinghouse_started(self) -> None: + # Lazily start the clearinghouse worker. + if not self._clearinghouse_started: + self._clearinghouse_started = True + self._executor.submit(self._clearinghouse_worker) + + def submit( + self, + key: str, + *args: Any, + done_callback: Optional[Callable[[Future], None]] = None, + ) -> None: + """See concurrent.futures.Executor#submit""" + + self._ensure_clearinghouse_started() + + self._pending_count.acquire() + self._pending.put(_BatchPartitionWorkItem(key, args, done_callback)) + + def shutdown(self) -> None: + if not self._clearinghouse_started: + # This is required to make shutdown() idempotent, which is important + # when it's called explicitly and then also by a context manager. + logger.debug("Shutting down: clearinghouse not started") + return + + logger.debug(f"Shutting down {self.__class__.__name__}") + + # Send the shutdown signal. + self._pending.put(None) + + # By acquiring all the permits, we ensure that no more tasks will be scheduled + # and automatically wait until all existing tasks have completed. + for _ in range(self.max_pending): + self._pending_count.acquire() + + # We must wait for the clearinghouse worker to exit before calling shutdown + # on the thread pool. Without this, the clearinghouse worker might fail to + # enqueue pending tasks into the pool. + while self._clearinghouse_started: + time.sleep(_PARTITION_EXECUTOR_FLUSH_SLEEP_INTERVAL) + + self._executor.shutdown(wait=False) + + def close(self) -> None: + self.shutdown() diff --git a/metadata-ingestion/src/datahub/utilities/prefix_batch_builder.py b/metadata-ingestion/src/datahub/utilities/prefix_batch_builder.py new file mode 100644 index 0000000000000..271c2517e8771 --- /dev/null +++ b/metadata-ingestion/src/datahub/utilities/prefix_batch_builder.py @@ -0,0 +1,81 @@ +import dataclasses +from collections import defaultdict +from typing import List + + +@dataclasses.dataclass +class PrefixGroup: + prefix: str + names: List[str] # every name in the list has the same prefix + exact_match: bool = False + + +def build_prefix_batches( + names: List[str], max_batch_size: int, max_groups_in_batch: int +) -> List[List[PrefixGroup]]: + """Split the names into a list of batches, where each batch is a list of groups and each group is a list of names with a common prefix.""" + + groups = _build_prefix_groups(names, max_batch_size=max_batch_size) + batches = _batch_prefix_groups( + groups, max_batch_size=max_batch_size, max_groups_in_batch=max_groups_in_batch + ) + return batches + + +def _build_prefix_groups(names: List[str], max_batch_size: int) -> List[PrefixGroup]: + """Given a list of names, group them by shared prefixes such that no group is larger than `max_batch_size`.""" + + def split_group(group: PrefixGroup) -> List[PrefixGroup]: + if len(group.names) <= max_batch_size: + return [group] + + result = [] + + # Split into subgroups by the next character. + prefix_length = len(group.prefix) + 1 + subgroups = defaultdict(list) + for name in group.names: + if len(name) < prefix_length: + # Handle cases where a single name is also the prefix for a large number of names. + # For example, if NAME and NAME_{1..10000} are both in the list. + result.append(PrefixGroup(prefix=name, names=[name], exact_match=True)) + continue + + prefix = name[:prefix_length] + subgroups[prefix].append(name) + + for prefix, names in subgroups.items(): + result.extend(split_group(PrefixGroup(prefix=prefix, names=names))) + + return result + + return split_group(PrefixGroup(prefix="", names=sorted(names))) + + +def _batch_prefix_groups( + groups: List[PrefixGroup], max_batch_size: int, max_groups_in_batch: int +) -> List[List[PrefixGroup]]: + """Batch the groups together, so that no batch's total is larger than `max_batch_size` + and no group in a batch is larger than `max_group_size`.""" + + # A batch is a set of groups. + + # This is a variant of the 1D bin packing problem, which is actually NP-hard. + # However, we'll just use a greedy algorithm for simplicity. + + batches = [] + current_batch_size = 0 + batch: List[PrefixGroup] = [] + for group in groups: + if ( + current_batch_size + len(group.names) > max_batch_size + or len(batch) > max_groups_in_batch + ): + batches.append(batch) + batch = [] + current_batch_size = 0 + batch.append(group) + current_batch_size += len(group.names) + if batch: + batches.append(batch) + return batches diff --git a/metadata-ingestion/src/datahub/utilities/serialized_lru_cache.py b/metadata-ingestion/src/datahub/utilities/serialized_lru_cache.py new file mode 100644 index 0000000000000..b5f490720340c --- /dev/null +++ b/metadata-ingestion/src/datahub/utilities/serialized_lru_cache.py @@ -0,0 +1,98 @@ +import functools +import threading +from typing import Callable, Dict, Hashable, Tuple, TypeVar + +import cachetools +import cachetools.keys +from typing_extensions import ParamSpec + +_Key = Tuple[Hashable, ...] +_F = ParamSpec("_F") +_T = TypeVar("_T") + + +def serialized_lru_cache( + maxsize: int, +) -> Callable[[Callable[_F, _T]], Callable[_F, _T]]: + """Similar to `lru_cache`, but ensures multiple calls with the same parameters are serialized. + + Calls with different parameters are allowed to proceed in parallel. + + Args: + maxsize (int): Maximum number of entries to keep in the cache. + + Returns: + Callable[[Callable[F, T]], Callable[F, T]]: Decorator for the function to be wrapped. + """ + + UNSET = object() + + def decorator(func: Callable[_F, _T]) -> Callable[_F, _T]: + hits = 0 + misses = 0 + + cache_lock = threading.Lock() + cache: "cachetools.LRUCache[_Key, _T]" = cachetools.LRUCache(maxsize=maxsize) + + key_locks_lock = threading.Lock() + key_locks: Dict[_Key, threading.Lock] = {} + key_waiters: Dict[_Key, int] = {} + + def wrapper(*args: _F.args, **kwargs: _F.kwargs) -> _T: + # We need a type ignore here because there's no way for us to require that + # the args and kwargs are hashable while using ParamSpec. + key: _Key = cachetools.keys.hashkey(*args, **{k: v for k, v in kwargs.items() if "cache_exclude" not in k}) # type: ignore + + with cache_lock: + if key in cache: + nonlocal hits + hits += 1 + return cache[key] + + with key_locks_lock: + if key not in key_locks: + key_locks[key] = threading.Lock() + key_waiters[key] = 0 + lock = key_locks[key] + key_waiters[key] += 1 + + try: + with lock: + # Check the cache again, in case the cache was updated by another thread. + result = UNSET + with cache_lock: + if key in cache: + hits += 1 + return cache[key] + + nonlocal misses + misses += 1 + result = func(*args, **kwargs) + + with cache_lock: + cache[key] = result + return result + + finally: + with key_locks_lock: + key_waiters[key] -= 1 + if key_waiters[key] == 0: + del key_locks[key] + del key_waiters[key] + + def cache_info() -> functools._CacheInfo: + return functools._CacheInfo( + hits=hits, + misses=misses, + maxsize=maxsize, + currsize=len(cache), + ) + + # Add some extra attributes to the wrapper function. This makes it mostly compatible + # with functools.lru_cache. + wrapper.cache = cache # type: ignore + wrapper.cache_info = cache_info # type: ignore + + return functools.update_wrapper(wrapper, func) + + return decorator diff --git a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json index b7e0c0169cccb..e7b2a7c4a9f4b 100644 --- a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json +++ b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json @@ -249,7 +249,11 @@ "nativeDataType": "INT", "recursive": false, "globalTags": { - "tags": [] + "tags": [ + { + "tag": "urn:li:tag:Test Policy Tag" + } + ] }, "glossaryTerms": { "terms": [ @@ -428,5 +432,21 @@ "runId": "bigquery-2022_02_03-07_00_00", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Test Policy Tag", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Test Policy Tag" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py b/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py index 26511d9e5df1a..a24b6174eb925 100644 --- a/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py +++ b/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py @@ -44,8 +44,10 @@ def random_email(): @patch.object(BigQuerySchemaApi, "get_columns_for_dataset") @patch.object(BigQueryDataReader, "get_sample_data_for_table") @patch("google.cloud.bigquery.Client") +@patch("google.cloud.datacatalog_v1.PolicyTagManagerClient") def test_bigquery_v2_ingest( client, + policy_tag_manager_client, get_sample_data_for_table, get_columns_for_dataset, get_datasets_for_project_id, @@ -78,6 +80,7 @@ def test_bigquery_v2_ingest( comment="comment", is_partition_column=False, cluster_column_position=None, + policy_tags=["Test Policy Tag"], ), BigqueryColumn( name="email", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json index 54f4309d962aa..aacb7093e5b4e 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json @@ -52,6 +52,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "language": "sql", "dbt_unique_id": "model.sample_dbt.customer_details", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -416,6 +417,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -643,6 +645,7 @@ "catalog_type": "VIEW", "language": "sql", "dbt_unique_id": "model.sample_dbt.payments_base", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -979,6 +982,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1170,6 +1174,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.actor", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1401,6 +1406,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.address", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1696,6 +1702,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.category", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1876,6 +1883,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.city", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2082,6 +2090,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.country", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2280,6 +2289,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.customer", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2621,6 +2631,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2874,6 +2885,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3141,6 +3153,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3390,6 +3403,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3639,6 +3653,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3888,6 +3903,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_06", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json index 04107bcde903e..5a35b4763af06 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json @@ -46,6 +46,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "language": "sql", "dbt_unique_id": "model.sample_dbt.customer_details", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -376,6 +377,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -596,6 +598,7 @@ "catalog_type": "VIEW", "language": "sql", "dbt_unique_id": "model.sample_dbt.payments_base", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -924,6 +927,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -1141,6 +1145,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "snapshot.sample_dbt.customer_snapshot", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -1541,6 +1546,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.actor", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -1766,6 +1772,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.address", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -2061,6 +2068,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.category", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -2241,6 +2249,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.city", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -2447,6 +2456,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.country", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -2645,6 +2655,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.customer", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -2986,6 +2997,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -3239,6 +3251,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -3506,6 +3519,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -3755,6 +3769,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -4004,6 +4019,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", @@ -4253,6 +4269,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_06", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json", "manifest_version": "1.3.0", "manifest_adapter": "postgres", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json index 4e5199aeec389..91095966eddd1 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json @@ -47,6 +47,7 @@ "catalog_type": "table", "language": "sql", "dbt_unique_id": "model.jaffle_shop.customers", + "dbt_package_name": "jaffle_shop", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", "manifest_adapter": "bigquery", @@ -271,6 +272,7 @@ "catalog_type": "table", "language": "sql", "dbt_unique_id": "model.jaffle_shop.orders", + "dbt_package_name": "jaffle_shop", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", "manifest_adapter": "bigquery", @@ -514,6 +516,7 @@ "catalog_type": "view", "language": "sql", "dbt_unique_id": "model.jaffle_shop.stg_customers", + "dbt_package_name": "jaffle_shop", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", "manifest_adapter": "bigquery", @@ -668,6 +671,7 @@ "catalog_type": "view", "language": "sql", "dbt_unique_id": "model.jaffle_shop.stg_orders", + "dbt_package_name": "jaffle_shop", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", "manifest_adapter": "bigquery", @@ -834,6 +838,7 @@ "catalog_type": "view", "language": "sql", "dbt_unique_id": "model.jaffle_shop.stg_payments", + "dbt_package_name": "jaffle_shop", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", "manifest_adapter": "bigquery", @@ -1000,6 +1005,7 @@ "catalog_type": "table", "language": "sql", "dbt_unique_id": "seed.jaffle_shop.raw_customers", + "dbt_package_name": "jaffle_shop", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", "manifest_adapter": "bigquery", @@ -1132,6 +1138,7 @@ "catalog_type": "table", "language": "sql", "dbt_unique_id": "seed.jaffle_shop.raw_orders", + "dbt_package_name": "jaffle_shop", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", "manifest_adapter": "bigquery", @@ -1276,6 +1283,7 @@ "catalog_type": "table", "language": "sql", "dbt_unique_id": "seed.jaffle_shop.raw_payments", + "dbt_package_name": "jaffle_shop", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", "manifest_adapter": "bigquery", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json index 60596547c3d50..a9b7df7c2b2a1 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json @@ -46,6 +46,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "language": "sql", "dbt_unique_id": "model.sample_dbt.customer_details", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -376,6 +377,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -722,6 +724,7 @@ "catalog_type": "VIEW", "language": "sql", "dbt_unique_id": "model.sample_dbt.payments_base", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -1180,6 +1183,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -1502,6 +1506,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "snapshot.sample_dbt.customer_snapshot", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -2027,6 +2032,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.actor", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -2249,6 +2255,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.address", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -2544,6 +2551,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.category", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -2724,6 +2732,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.city", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -2930,6 +2939,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.country", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -3125,6 +3135,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.customer", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -3466,6 +3477,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -3719,6 +3731,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -3983,6 +3996,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -4232,6 +4246,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -4481,6 +4496,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -4730,6 +4746,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_06", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", "manifest_version": "1.7.3", "manifest_adapter": "postgres", @@ -5993,6 +6010,70 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:36225e795a4597b2376996774a803b0d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:57aa623f096cf3a28af70fe94b713907", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:a42a5b1bee156e45972e12d4156fb7a2", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:fc6268f0be68fd04c310705b65efd6fe", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:dbt:tag_from_dbt", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json index 4ec48e71badcd..4863a1e4f7398 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json @@ -51,6 +51,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "language": "sql", "dbt_unique_id": "model.sample_dbt.customer_details", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -380,6 +381,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -589,6 +591,7 @@ "catalog_type": "VIEW", "language": "sql", "dbt_unique_id": "model.sample_dbt.payments_base", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -925,6 +928,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1116,6 +1120,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.actor", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1344,6 +1349,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.address", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1639,6 +1645,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.category", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1819,6 +1826,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.city", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2025,6 +2033,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.country", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2220,6 +2229,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.customer", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2561,6 +2571,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2814,6 +2825,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3078,6 +3090,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3327,6 +3340,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3576,6 +3590,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json index 9002001fde29e..7c61e9fcd0e3a 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json @@ -53,6 +53,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "language": "sql", "dbt_unique_id": "model.sample_dbt.customer_details", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -383,6 +384,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -593,6 +595,7 @@ "catalog_type": "VIEW", "language": "sql", "dbt_unique_id": "model.sample_dbt.payments_base", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -930,6 +933,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1122,6 +1126,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.actor", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1351,6 +1356,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.address", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1647,6 +1653,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.category", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1828,6 +1835,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.city", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2035,6 +2043,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.country", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2231,6 +2240,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.customer", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2573,6 +2583,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2827,6 +2838,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3092,6 +3104,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3342,6 +3355,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3592,6 +3606,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3842,6 +3857,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_06", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json index d16542adaa030..a2a8437d551eb 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json @@ -52,6 +52,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "language": "sql", "dbt_unique_id": "model.sample_dbt.customer_details", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -381,6 +382,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -590,6 +592,7 @@ "catalog_type": "VIEW", "language": "sql", "dbt_unique_id": "model.sample_dbt.payments_base", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -926,6 +929,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1117,6 +1121,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.actor", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1345,6 +1350,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.address", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1640,6 +1646,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.category", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1820,6 +1827,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.city", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2026,6 +2034,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.country", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2221,6 +2230,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.customer", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2562,6 +2572,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2815,6 +2826,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3079,6 +3091,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3328,6 +3341,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3577,6 +3591,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3826,6 +3841,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_06", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json index 921af5cb3d1db..c37f3847117f6 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json @@ -52,6 +52,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "language": "sql", "dbt_unique_id": "model.sample_dbt.customer_details", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -381,6 +382,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -590,6 +592,7 @@ "catalog_type": "VIEW", "language": "sql", "dbt_unique_id": "model.sample_dbt.payments_base", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -926,6 +929,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1117,6 +1121,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.actor", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1345,6 +1350,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.address", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1640,6 +1646,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.category", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -1820,6 +1827,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.city", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2026,6 +2034,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.country", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2221,6 +2230,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.customer", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2562,6 +2572,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -2815,6 +2826,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3079,6 +3091,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3328,6 +3341,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3577,6 +3591,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", @@ -3826,6 +3841,7 @@ "catalog_type": "BASE TABLE", "language": "sql", "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_06", + "dbt_package_name": "sample_dbt", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", "manifest_adapter": "postgres", diff --git a/metadata-ingestion/tests/integration/fivetran/fivetran_bigquery_golden.json b/metadata-ingestion/tests/integration/fivetran/fivetran_bigquery_golden.json deleted file mode 100644 index fcf354d7a1405..0000000000000 --- a/metadata-ingestion/tests/integration/fivetran/fivetran_bigquery_golden.json +++ /dev/null @@ -1,628 +0,0 @@ -[ -{ - "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(fivetran,calendar_elected,PROD)", - "changeType": "UPSERT", - "aspectName": "dataFlowInfo", - "aspect": { - "json": { - "customProperties": {}, - "name": "postgres" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(fivetran,calendar_elected,PROD)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:fivetran" - } - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(fivetran,calendar_elected,PROD)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)", - "changeType": "UPSERT", - "aspectName": "dataJobInfo", - "aspect": { - "json": { - "customProperties": { - "paused": "False", - "sync_frequency": "1440", - "destination_id": "'interval_unconstitutional'" - }, - "name": "postgres", - "type": { - "string": "COMMAND" - } - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)", - "changeType": "UPSERT", - "aspectName": "dataJobInputOutput", - "aspect": { - "json": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV)", - "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:bigquery,test.postgres_public.employee,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:bigquery,test.postgres_public.company,PROD)" - ], - "inputDatajobs": [], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV),id)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,test.postgres_public.employee,PROD),id)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV),name)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,test.postgres_public.employee,PROD),name)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV),id)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,test.postgres_public.company,PROD),id)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV),name)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,test.postgres_public.company,PROD),name)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:abc.xyz@email.com", - "type": "DEVELOPER", - "source": { - "type": "SERVICE" - } - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:fivetran" - } - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:ee88d32dbe3133a23a9023c097050190", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceProperties", - "aspect": { - "json": { - "customProperties": {}, - "name": "4c9a03d6-eded-4422-a46a-163266e58243", - "type": "BATCH_SCHEDULED", - "created": { - "time": 1695191853000, - "actor": "urn:li:corpuser:datahub" - } - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:ee88d32dbe3133a23a9023c097050190", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRelationships", - "aspect": { - "json": { - "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)", - "upstreamInstances": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:ee88d32dbe3133a23a9023c097050190", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceInput", - "aspect": { - "json": { - "inputs": [ - "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV)", - "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV)" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:ee88d32dbe3133a23a9023c097050190", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceOutput", - "aspect": { - "json": { - "outputs": [ - "urn:li:dataset:(urn:li:dataPlatform:bigquery,test.postgres_public.employee,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:bigquery,test.postgres_public.company,PROD)" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:ee88d32dbe3133a23a9023c097050190", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRunEvent", - "aspect": { - "json": { - "timestampMillis": 1695191853000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "status": "STARTED" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:ee88d32dbe3133a23a9023c097050190", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRunEvent", - "aspect": { - "json": { - "timestampMillis": 1695191885000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "status": "COMPLETE", - "result": { - "type": "SUCCESS", - "nativeResultType": "fivetran" - } - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceProperties", - "aspect": { - "json": { - "customProperties": {}, - "name": "f773d1e9-c791-48f4-894f-8cf9b3dfc834", - "type": "BATCH_SCHEDULED", - "created": { - "time": 1696343730000, - "actor": "urn:li:corpuser:datahub" - } - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRelationships", - "aspect": { - "json": { - "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)", - "upstreamInstances": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceInput", - "aspect": { - "json": { - "inputs": [ - "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV)", - "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV)" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceOutput", - "aspect": { - "json": { - "outputs": [ - "urn:li:dataset:(urn:li:dataPlatform:bigquery,test.postgres_public.employee,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:bigquery,test.postgres_public.company,PROD)" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRunEvent", - "aspect": { - "json": { - "timestampMillis": 1696343730000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "status": "STARTED" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRunEvent", - "aspect": { - "json": { - "timestampMillis": 1696343732000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "status": "COMPLETE", - "result": { - "type": "SKIPPED", - "nativeResultType": "fivetran" - } - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:d8f100271d2dc3fa905717f82d083c8d", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceProperties", - "aspect": { - "json": { - "customProperties": {}, - "name": "63c2fc85-600b-455f-9ba0-f576522465be", - "type": "BATCH_SCHEDULED", - "created": { - "time": 1696343755000, - "actor": "urn:li:corpuser:datahub" - } - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:d8f100271d2dc3fa905717f82d083c8d", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRelationships", - "aspect": { - "json": { - "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)", - "upstreamInstances": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:d8f100271d2dc3fa905717f82d083c8d", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceInput", - "aspect": { - "json": { - "inputs": [ - "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.employee,DEV)", - "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres_db.public.company,DEV)" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:d8f100271d2dc3fa905717f82d083c8d", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceOutput", - "aspect": { - "json": { - "outputs": [ - "urn:li:dataset:(urn:li:dataPlatform:bigquery,test.postgres_public.employee,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:bigquery,test.postgres_public.company,PROD)" - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:d8f100271d2dc3fa905717f82d083c8d", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRunEvent", - "aspect": { - "json": { - "timestampMillis": 1696343755000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "status": "STARTED" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataProcessInstance", - "entityUrn": "urn:li:dataProcessInstance:d8f100271d2dc3fa905717f82d083c8d", - "changeType": "UPSERT", - "aspectName": "dataProcessInstanceRunEvent", - "aspect": { - "json": { - "timestampMillis": 1696343790000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "status": "COMPLETE", - "result": { - "type": "FAILURE", - "nativeResultType": "fivetran" - } - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(fivetran,calendar_elected,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(fivetran,calendar_elected,PROD),calendar_elected)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -} -] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_empty_connection_user_golden.json b/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_empty_connection_user_golden.json index 65f8620d0b4c4..d2ae437605644 100644 --- a/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_empty_connection_user_golden.json +++ b/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_empty_connection_user_golden.json @@ -24,6 +24,7 @@ "aspect": { "json": { "owners": [], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:fivetran" @@ -186,6 +187,7 @@ "aspect": { "json": { "owners": [], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:fivetran" @@ -614,5 +616,53 @@ "runId": "powerbi-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:d8f100271d2dc3fa905717f82d083c8d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:ee88d32dbe3133a23a9023c097050190", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_golden.json b/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_golden.json index 8545f43348075..59e545183a4ec 100644 --- a/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_golden.json +++ b/metadata-ingestion/tests/integration/fivetran/fivetran_snowflake_golden.json @@ -624,5 +624,53 @@ "runId": "powerbi-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:be36f55c13ec4e313c7510770e50784a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:d8f100271d2dc3fa905717f82d083c8d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:ee88d32dbe3133a23a9023c097050190", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/fivetran/test_fivetran.py b/metadata-ingestion/tests/integration/fivetran/test_fivetran.py index de1e5543f4be6..887dcce4b7e9b 100644 --- a/metadata-ingestion/tests/integration/fivetran/test_fivetran.py +++ b/metadata-ingestion/tests/integration/fivetran/test_fivetran.py @@ -7,12 +7,14 @@ from freezegun import freeze_time from datahub.configuration.common import ConfigurationWarning +from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.run.pipeline import Pipeline from datahub.ingestion.source.fivetran.config import ( BigQueryDestinationConfig, FivetranSourceConfig, SnowflakeDestinationConfig, ) +from datahub.ingestion.source.fivetran.fivetran import FivetranSource from datahub.ingestion.source.fivetran.fivetran_query import FivetranLogQuery from datahub.ingestion.source_config.usage.bigquery_usage import BigQueryCredential from tests.test_helpers import mce_helpers @@ -89,7 +91,7 @@ def default_query_results( "destination_column_name": "name", }, ] - elif query == fivetran_log_query.get_user_query("reapply_phone"): + elif query == fivetran_log_query.get_users_query(): return [ { "user_id": "reapply_phone", @@ -98,7 +100,9 @@ def default_query_results( "email": "abc.xyz@email.com", } ] - elif query == fivetran_log_query.get_sync_logs_query(): + elif query == fivetran_log_query.get_sync_logs_query().format( + db_clause=fivetran_log_query.db_clause, syncs_interval=7 + ): return [ { "connector_id": "calendar_elected", @@ -318,73 +322,28 @@ def test_fivetran_with_snowflake_dest_and_null_connector_user(pytestconfig, tmp_ @freeze_time(FROZEN_TIME) @pytest.mark.integration -def test_fivetran_with_bigquery_dest(pytestconfig, tmp_path): - test_resources_dir = pytestconfig.rootpath / "tests/integration/fivetran" - - # Run the metadata ingestion pipeline. - output_file = tmp_path / "fivetran_test_events.json" - golden_file = test_resources_dir / "fivetran_bigquery_golden.json" - - with mock.patch( - "datahub.ingestion.source.fivetran.fivetran_log_api.create_engine" - ) as mock_create_engine: - connection_magic_mock = MagicMock() - connection_magic_mock.execute.side_effect = default_query_results - - mock_create_engine.return_value = connection_magic_mock - - pipeline = Pipeline.create( +def test_fivetran_bigquery_config(): + with mock.patch("datahub.ingestion.source.fivetran.fivetran_log_api.create_engine"): + # Simply test that the config is parsed and the source is initialized without an error. + assert FivetranSource.create( { - "run_id": "powerbi-test", - "source": { - "type": "fivetran", - "config": { - "fivetran_log_config": { - "destination_platform": "bigquery", - "bigquery_destination_config": { - "credential": { - "private_key_id": "testprivatekey", - "project_id": "test-project", - "client_email": "fivetran-connector@test-project.iam.gserviceaccount.com", - "client_id": "1234567", - "private_key": "private-key", - }, - "dataset": "test", - }, - }, - "connector_patterns": { - "allow": [ - "postgres", - ] - }, - "sources_to_database": { - "calendar_elected": "postgres_db", - }, - "sources_to_platform_instance": { - "calendar_elected": { - "env": "DEV", - } + "fivetran_log_config": { + "destination_platform": "bigquery", + "bigquery_destination_config": { + "credential": { + "private_key_id": "testprivatekey", + "project_id": "test-project", + "client_email": "fivetran-connector@test-project.iam.gserviceaccount.com", + "client_id": "1234567", + "private_key": "private-key", }, + "dataset": "test", }, }, - "sink": { - "type": "file", - "config": { - "filename": f"{output_file}", - }, - }, - } + }, + ctx=PipelineContext(run_id="fivetran-bq-dummy"), ) - pipeline.run() - pipeline.raise_from_status() - - mce_helpers.check_golden_file( - pytestconfig, - output_path=f"{output_file}", - golden_path=f"{golden_file}", - ) - @freeze_time(FROZEN_TIME) def test_fivetran_snowflake_destination_config(): diff --git a/metadata-ingestion/tests/integration/grafana/default-dashboard.json b/metadata-ingestion/tests/integration/grafana/default-dashboard.json new file mode 100644 index 0000000000000..8ce40ad6acb13 --- /dev/null +++ b/metadata-ingestion/tests/integration/grafana/default-dashboard.json @@ -0,0 +1,25 @@ +{ + "id": null, + "uid": "default", + "title": "Default Dashboard", + "tags": [], + "timezone": "browser", + "schemaVersion": 16, + "version": 0, + "panels": [ + { + "type": "text", + "title": "Welcome", + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 5 + }, + "options": { + "content": "Welcome to your Grafana dashboard!", + "mode": "markdown" + } + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/grafana/docker-compose.yml b/metadata-ingestion/tests/integration/grafana/docker-compose.yml new file mode 100644 index 0000000000000..41995a1d49da6 --- /dev/null +++ b/metadata-ingestion/tests/integration/grafana/docker-compose.yml @@ -0,0 +1,32 @@ +version: '3.7' + +services: + grafana: + image: grafana/grafana:latest + container_name: grafana + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_SECURITY_ADMIN_USER=admin + - GF_PATHS_PROVISIONING=/etc/grafana/provisioning + volumes: + - grafana-storage:/var/lib/grafana + - ./provisioning:/etc/grafana/provisioning + - ./default-dashboard.json:/var/lib/grafana/dashboards/default-dashboard.json + depends_on: + - postgres + + postgres: + image: postgres:13 + container_name: grafana-postgres + environment: + POSTGRES_DB: grafana + POSTGRES_USER: grafana + POSTGRES_PASSWORD: grafana + volumes: + - postgres-storage:/var/lib/postgresql/data + +volumes: + grafana-storage: + postgres-storage: diff --git a/metadata-ingestion/tests/integration/grafana/grafana_mcps_golden.json b/metadata-ingestion/tests/integration/grafana/grafana_mcps_golden.json new file mode 100644 index 0000000000000..1447e840eac8c --- /dev/null +++ b/metadata-ingestion/tests/integration/grafana/grafana_mcps_golden.json @@ -0,0 +1,56 @@ +[ +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(grafana,default)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "displayName": "Default Dashboard", + "id": "1", + "uid": "default", + "title": "Default Dashboard", + "uri": "db/default-dashboard", + "type": "dash-db" + }, + "externalUrl": "http://localhost:3000/d/default/default-dashboard", + "title": "Default Dashboard", + "description": "", + "charts": [], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test-simple", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(grafana,default)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test-simple", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/grafana/provisioning/api-keys/api_keys.yaml b/metadata-ingestion/tests/integration/grafana/provisioning/api-keys/api_keys.yaml new file mode 100644 index 0000000000000..7ef096b6bfe97 --- /dev/null +++ b/metadata-ingestion/tests/integration/grafana/provisioning/api-keys/api_keys.yaml @@ -0,0 +1,3 @@ +api_keys: + - name: 'example-api-key' + role: 'Admin' diff --git a/metadata-ingestion/tests/integration/grafana/provisioning/dashboards/dashboard.yaml b/metadata-ingestion/tests/integration/grafana/provisioning/dashboards/dashboard.yaml new file mode 100644 index 0000000000000..e6d4aa3a45a39 --- /dev/null +++ b/metadata-ingestion/tests/integration/grafana/provisioning/dashboards/dashboard.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards diff --git a/metadata-ingestion/tests/integration/grafana/provisioning/datasources/datasource.yaml b/metadata-ingestion/tests/integration/grafana/provisioning/datasources/datasource.yaml new file mode 100644 index 0000000000000..9ba65ec1a54bc --- /dev/null +++ b/metadata-ingestion/tests/integration/grafana/provisioning/datasources/datasource.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +datasources: + - name: PostgreSQL + type: postgres + access: proxy + url: postgres:5432 + database: grafana + user: grafana + password: grafana + jsonData: + sslmode: disable diff --git a/metadata-ingestion/tests/integration/grafana/provisioning/service_accounts/service_accounts.yaml b/metadata-ingestion/tests/integration/grafana/provisioning/service_accounts/service_accounts.yaml new file mode 100644 index 0000000000000..a6c259aac77ab --- /dev/null +++ b/metadata-ingestion/tests/integration/grafana/provisioning/service_accounts/service_accounts.yaml @@ -0,0 +1,6 @@ +service_accounts: + - name: 'example-service-account' + role: 'Admin' + apiKeys: + - keyName: 'example-api-key' + role: 'Admin' diff --git a/metadata-ingestion/tests/integration/grafana/test_grafana.py b/metadata-ingestion/tests/integration/grafana/test_grafana.py new file mode 100644 index 0000000000000..6eb6b0b850926 --- /dev/null +++ b/metadata-ingestion/tests/integration/grafana/test_grafana.py @@ -0,0 +1,191 @@ +import logging +import time +from base64 import b64encode + +import pytest +import requests +from freezegun import freeze_time + +from datahub.ingestion.run.pipeline import Pipeline +from tests.test_helpers import fs_helpers, mce_helpers +from tests.test_helpers.docker_helpers import cleanup_image, wait_for_port + +pytestmark = pytest.mark.integration_batch_2 + +FROZEN_TIME = "2024-07-12 12:00:00" + + +logger = logging.getLogger(__name__) + + +class GrafanaClient: + def __init__(self, url, admin_user, admin_password): + self.url = url + self.auth = (admin_user, admin_password) + self.headers = { + "Authorization": f"Basic {b64encode(f'{admin_user}:{admin_password}'.encode()).decode()}", + "Content-Type": "application/json", + } + + def create_service_account(self, name, role): + service_account_payload = {"name": name, "role": role, "isDisabled": False} + try: + response = requests.post( + f"{self.url}/api/serviceaccounts", + headers=self.headers, + json=service_account_payload, + ) + response.raise_for_status() + service_account = response.json() + return service_account + except requests.exceptions.RequestException as e: + logging.error(f"Error creating service account: {e}") + return None + + def create_api_key(self, service_account_id, key_name, role): + api_key_payload = {"name": key_name, "role": role} + try: + response = requests.post( + f"{self.url}/api/serviceaccounts/{service_account_id}/tokens", + headers=self.headers, + json=api_key_payload, + ) + response.raise_for_status() + api_key = response.json() + return api_key["key"] + except requests.exceptions.RequestException as e: + logging.error(f"Error creating API key: {e}") + return None + + +@pytest.fixture(scope="module") +def test_resources_dir(pytestconfig): + return pytestconfig.rootpath / "tests/integration/grafana" + + +@pytest.fixture(scope="module") +def test_api_key(): + # Example usage: + url = "http://localhost:3000" + admin_user = "admin" + admin_password = "admin" + + grafana_client = GrafanaClient(url, admin_user, admin_password) + + # Step 1: Create the service account + service_account = grafana_client.create_service_account( + name="example-service-account", role="Viewer" + ) + if service_account: + print(f"Service Account Created: {service_account}") + + # Step 2: Create the API key for the service account + api_key = grafana_client.create_api_key( + service_account_id=service_account["id"], + key_name="example-api-key", + role="Admin", + ) + if api_key: + print("Service Account API Key:", api_key) + return api_key + else: + print("Failed to create API key for the service account") + else: + print("Failed to create service account") + + +@pytest.fixture(scope="module") +def loaded_grafana(docker_compose_runner, test_resources_dir): + with docker_compose_runner( + test_resources_dir / "docker-compose.yml", "grafana" + ) as docker_services: + wait_for_port( + docker_services, + container_name="grafana", + container_port=3000, + timeout=300, + ) + yield docker_services + + # The Grafana image can be large, so we remove it after the test. + cleanup_image("grafana/grafana") + + +@freeze_time(FROZEN_TIME) +def test_grafana_dashboard(loaded_grafana, pytestconfig, tmp_path, test_resources_dir): + # Wait for Grafana to be up and running + url = "http://localhost:3000/api/health" + for i in range(30): + logging.info("waiting for Grafana to start...") + time.sleep(5) + resp = requests.get(url) + if resp.status_code == 200: + logging.info(f"Grafana started after waiting {i*5} seconds") + break + else: + pytest.fail("Grafana did not start in time") + + # Check if the default dashboard is loaded + dashboard_url = "http://localhost:3000/api/dashboards/uid/default" + resp = requests.get(dashboard_url, auth=("admin", "admin")) + assert resp.status_code == 200, "Failed to load default dashboard" + dashboard = resp.json() + + assert ( + dashboard["dashboard"]["title"] == "Default Dashboard" + ), "Default dashboard title mismatch" + assert any( + panel["type"] == "text" for panel in dashboard["dashboard"]["panels"] + ), "Default dashboard missing text panel" + + # Verify the output. (You can add further checks here if needed) + logging.info("Default dashboard verified successfully") + + +@freeze_time(FROZEN_TIME) +def test_grafana_ingest( + loaded_grafana, pytestconfig, tmp_path, test_resources_dir, test_api_key +): + # Wait for Grafana to be up and running + url = "http://localhost:3000/api/health" + for i in range(30): + logging.info("waiting for Grafana to start...") + time.sleep(5) + resp = requests.get(url) + if resp.status_code == 200: + logging.info(f"Grafana started after waiting {i*5} seconds") + break + else: + pytest.fail("Grafana did not start in time") + + # Run the metadata ingestion pipeline. + with fs_helpers.isolated_filesystem(tmp_path): + # Run grafana ingestion run. + pipeline = Pipeline.create( + { + "run_id": "grafana-test-simple", + "source": { + "type": "grafana", + "config": { + "url": "http://localhost:3000", + "service_account_token": test_api_key, + }, + }, + "sink": { + "type": "file", + "config": {"filename": "./grafana_mcps.json"}, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() + + # Verify the output. + mce_helpers.check_golden_file( + pytestconfig, + output_path="grafana_mcps.json", + golden_path=test_resources_dir / "grafana_mcps_golden.json", + ignore_paths=[ + r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['last_event_time'\]", + ], + ) diff --git a/metadata-ingestion/tests/integration/hive-metastore/docker-compose.yml b/metadata-ingestion/tests/integration/hive-metastore/docker-compose.yml index bc52779d71cf8..d6a6701451ff1 100644 --- a/metadata-ingestion/tests/integration/hive-metastore/docker-compose.yml +++ b/metadata-ingestion/tests/integration/hive-metastore/docker-compose.yml @@ -1,9 +1,6 @@ # Adapted from https://github.com/big-data-europe/docker-hive. -version: "3" - services: - presto: image: starburstdata/presto:350-e.15 container_name: "presto" diff --git a/metadata-ingestion/tests/integration/iceberg/docker-compose.yml b/metadata-ingestion/tests/integration/iceberg/docker-compose.yml index ab5c534e7289b..8baae6e8ab636 100644 --- a/metadata-ingestion/tests/integration/iceberg/docker-compose.yml +++ b/metadata-ingestion/tests/integration/iceberg/docker-compose.yml @@ -19,7 +19,7 @@ services: - AWS_REGION=us-east-1 ports: - 8888:8888 - - 8080:8080 + - 28080:8080 - 10000:10000 - 10001:10001 rest: diff --git a/metadata-ingestion/tests/integration/iceberg/iceberg_deleted_table_mces_golden.json b/metadata-ingestion/tests/integration/iceberg/iceberg_deleted_table_mces_golden.json index cc94625560a43..3321fcac0d73e 100644 --- a/metadata-ingestion/tests/integration/iceberg/iceberg_deleted_table_mces_golden.json +++ b/metadata-ingestion/tests/integration/iceberg/iceberg_deleted_table_mces_golden.json @@ -1,184 +1,192 @@ [ - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,test_platform_instance.nyc.another_taxis,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "owner": "root", - "created-at": "2023-07-04T14:23:10.457317300Z", - "write.format.default": "parquet", - "location": "s3a://warehouse/wh/nyc/another_taxis", - "format-version": "1", - "snapshot-id": "6904764113937987369", - "manifest-list": "s3a://warehouse/wh/nyc/another_taxis/metadata/snap-6904764113937987369-1-f18ce54a-d59c-461a-a066-9d3085ccf2f2.avro" +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,test_platform_instance.nyc.another_taxis,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "owner": "root", + "created-at": "2024-06-27T17:29:32.492204247Z", + "write.format.default": "parquet", + "location": "s3a://warehouse/wh/nyc/another_taxis", + "format-version": "1", + "partition-spec": "[{\"name\": \"trip_date\", \"transform\": \"identity\", \"source\": \"trip_date\", \"source-id\": 2, \"source-type\": \"timestamptz\", \"field-id\": 1000}]", + "snapshot-id": "1131595459662979239", + "manifest-list": "s3a://warehouse/wh/nyc/another_taxis/metadata/snap-1131595459662979239-1-0e80739b-774c-4eda-9d96-3a4c70873c32.avro" + }, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:root", + "type": "TECHNICAL_OWNER" }, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:root", - "type": "TECHNICAL_OWNER" - }, - { - "owner": "urn:li:corpGroup:root", - "type": "TECHNICAL_OWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + { + "owner": "urn:li:corpGroup:root", + "type": "TECHNICAL_OWNER" } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "nyc.another_taxis", - "platform": "urn:li:dataPlatform:iceberg", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "table {\n 1: vendor_id: optional long\n 2: trip_date: optional timestamptz\n 3: trip_id: optional long\n 4: trip_distance: optional float\n 5: fare_amount: optional double\n 6: store_and_fwd_flag: optional string\n}" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=struct].[type=long].vendor_id", - "nullable": true, + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "nyc.another_taxis", + "platform": "urn:li:dataPlatform:iceberg", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "table {\n 1: vendor_id: optional long\n 2: trip_date: optional timestamptz\n 3: trip_id: optional long\n 4: trip_distance: optional float\n 5: fare_amount: optional double\n 6: store_and_fwd_flag: optional string\n}" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=struct].[type=long].vendor_id", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "long", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=long].trip_date", - "nullable": true, + "nativeDataType": "long", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=long].trip_date", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamptz", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"logicalType\": \"timestamp-micros\", \"native_data_type\": \"timestamptz\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.TimeType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=long].trip_id", - "nullable": true, + "nativeDataType": "timestamptz", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"logicalType\": \"timestamp-micros\", \"native_data_type\": \"timestamptz\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=long].trip_id", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "long", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=float].trip_distance", - "nullable": true, + "nativeDataType": "long", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=float].trip_distance", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "float", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=double].fare_amount", - "nullable": true, + "nativeDataType": "float", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=double].fare_amount", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "double", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"double\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=string].store_and_fwd_flag", - "nullable": true, + "nativeDataType": "double", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"double\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=string].store_and_fwd_flag", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "iceberg-2020_04_14-07_00_00" + } + ] } }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,test_platform_instance.nyc.another_taxis,PROD)", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:iceberg", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:iceberg,test_platform_instance)" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "iceberg-2020_04_14-07_00_00" + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "iceberg-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,test_platform_instance.nyc.another_taxis,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:iceberg", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:iceberg,test_platform_instance)" } }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,test_platform_instance.nyc.taxis,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": true - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "iceberg-2020_04_14-07_00_00" + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "iceberg-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,test_platform_instance.nyc.taxis,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": true } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "iceberg-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } - ] \ No newline at end of file +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/iceberg/iceberg_ingest_mces_golden.json b/metadata-ingestion/tests/integration/iceberg/iceberg_ingest_mces_golden.json index 163911623470e..b017b6cd31520 100644 --- a/metadata-ingestion/tests/integration/iceberg/iceberg_ingest_mces_golden.json +++ b/metadata-ingestion/tests/integration/iceberg/iceberg_ingest_mces_golden.json @@ -1,153 +1,156 @@ [ - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,nyc.taxis,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "owner": "root", - "created-at": "2023-06-12T17:32:17.227545005Z", - "write.format.default": "parquet", - "location": "s3a://warehouse/wh/nyc/taxis", - "format-version": "1", - "snapshot-id": "2505818429184337337", - "manifest-list": "s3a://warehouse/wh/nyc/taxis/metadata/snap-2505818429184337337-1-a64915c4-afc8-40e3-97a7-98b072b42e10.avro" +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,nyc.taxis,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "owner": "root", + "created-at": "2024-05-22T14:08:04.001538500Z", + "write.format.default": "parquet", + "location": "s3a://warehouse/wh/nyc/taxis", + "format-version": "1", + "partition-spec": "[{\"name\": \"trip_date\", \"transform\": \"identity\", \"source\": \"trip_date\", \"source-id\": 2, \"source-type\": \"timestamptz\", \"field-id\": 1000}]", + "snapshot-id": "5259199139271057622", + "manifest-list": "s3a://warehouse/wh/nyc/taxis/metadata/snap-5259199139271057622-1-24dca7b8-d437-458e-ae91-df1d3e30bdc8.avro" + }, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:root", + "type": "TECHNICAL_OWNER" }, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:root", - "type": "TECHNICAL_OWNER" - }, - { - "owner": "urn:li:corpGroup:root", - "type": "TECHNICAL_OWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + { + "owner": "urn:li:corpGroup:root", + "type": "TECHNICAL_OWNER" } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "nyc.taxis", - "platform": "urn:li:dataPlatform:iceberg", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "table {\n 1: vendor_id: optional long\n 2: trip_date: optional timestamptz\n 3: trip_id: optional long\n 4: trip_distance: optional float\n 5: fare_amount: optional double\n 6: store_and_fwd_flag: optional string\n}" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=struct].[type=long].vendor_id", - "nullable": true, + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "nyc.taxis", + "platform": "urn:li:dataPlatform:iceberg", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "table {\n 1: vendor_id: optional long\n 2: trip_date: optional timestamptz\n 3: trip_id: optional long\n 4: trip_distance: optional float\n 5: fare_amount: optional double\n 6: store_and_fwd_flag: optional string\n}" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=struct].[type=long].vendor_id", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "long", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=long].trip_date", - "nullable": true, + "nativeDataType": "long", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=long].trip_date", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamptz", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"logicalType\": \"timestamp-micros\", \"native_data_type\": \"timestamptz\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.TimeType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=long].trip_id", - "nullable": true, + "nativeDataType": "timestamptz", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"logicalType\": \"timestamp-micros\", \"native_data_type\": \"timestamptz\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=long].trip_id", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "long", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=float].trip_distance", - "nullable": true, + "nativeDataType": "long", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=float].trip_distance", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "float", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=double].fare_amount", - "nullable": true, + "nativeDataType": "float", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=double].fare_amount", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "double", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"double\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=string].store_and_fwd_flag", - "nullable": true, + "nativeDataType": "double", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"double\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=string].store_and_fwd_flag", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "iceberg-test" + } + ] } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "iceberg-test", + "lastRunId": "no-run-id-provided" } - ] \ No newline at end of file +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/iceberg/iceberg_profile_mces_golden.json b/metadata-ingestion/tests/integration/iceberg/iceberg_profile_mces_golden.json index bdb7091014626..453a79494fa25 100644 --- a/metadata-ingestion/tests/integration/iceberg/iceberg_profile_mces_golden.json +++ b/metadata-ingestion/tests/integration/iceberg/iceberg_profile_mces_golden.json @@ -1,216 +1,220 @@ [ - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,nyc.taxis,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "owner": "root", - "created-at": "2023-06-12T17:33:25.422993540Z", - "write.format.default": "parquet", - "location": "s3a://warehouse/wh/nyc/taxis", - "format-version": "1", - "snapshot-id": "2585047006374307840", - "manifest-list": "s3a://warehouse/wh/nyc/taxis/metadata/snap-2585047006374307840-1-2e2bef19-40d1-4ad1-8fad-e57783477710.avro" +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,nyc.taxis,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "owner": "root", + "created-at": "2024-05-22T14:10:22.926080700Z", + "write.format.default": "parquet", + "location": "s3a://warehouse/wh/nyc/taxis", + "format-version": "1", + "partition-spec": "[{\"name\": \"trip_date\", \"transform\": \"identity\", \"source\": \"trip_date\", \"source-id\": 2, \"source-type\": \"timestamptz\", \"field-id\": 1000}]", + "snapshot-id": "564034874306625146", + "manifest-list": "s3a://warehouse/wh/nyc/taxis/metadata/snap-564034874306625146-1-562a1705-d774-4e0a-baf0-1988bcc7be72.avro" + }, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:root", + "type": "TECHNICAL_OWNER" }, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:root", - "type": "TECHNICAL_OWNER" - }, - { - "owner": "urn:li:corpGroup:root", - "type": "TECHNICAL_OWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + { + "owner": "urn:li:corpGroup:root", + "type": "TECHNICAL_OWNER" } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "nyc.taxis", - "platform": "urn:li:dataPlatform:iceberg", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "table {\n 1: vendor_id: optional long\n 2: trip_date: optional timestamptz\n 3: trip_id: optional long\n 4: trip_distance: optional float\n 5: fare_amount: optional double\n 6: store_and_fwd_flag: optional string\n}" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=struct].[type=long].vendor_id", - "nullable": true, + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "nyc.taxis", + "platform": "urn:li:dataPlatform:iceberg", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "table {\n 1: vendor_id: optional long\n 2: trip_date: optional timestamptz\n 3: trip_id: optional long\n 4: trip_distance: optional float\n 5: fare_amount: optional double\n 6: store_and_fwd_flag: optional string\n}" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=struct].[type=long].vendor_id", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "long", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=long].trip_date", - "nullable": true, + "nativeDataType": "long", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=long].trip_date", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamptz", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"logicalType\": \"timestamp-micros\", \"native_data_type\": \"timestamptz\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.TimeType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=long].trip_id", - "nullable": true, + "nativeDataType": "timestamptz", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"logicalType\": \"timestamp-micros\", \"native_data_type\": \"timestamptz\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=long].trip_id", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "long", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=float].trip_distance", - "nullable": true, + "nativeDataType": "long", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=float].trip_distance", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "float", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=double].fare_amount", - "nullable": true, + "nativeDataType": "float", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=double].fare_amount", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "double", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"double\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=string].store_and_fwd_flag", - "nullable": true, + "nativeDataType": "double", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"double\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=string].store_and_fwd_flag", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "iceberg-test" + } + ] } }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,nyc.taxis,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1586847600000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "iceberg-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,nyc.taxis,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1586847600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 5, + "columnCount": 6, + "fieldProfiles": [ + { + "fieldPath": "vendor_id", + "nullCount": 0, + "nullProportion": 0.0, + "min": "1", + "max": "3" }, - "rowCount": 5, - "columnCount": 6, - "fieldProfiles": [ - { - "fieldPath": "vendor_id", - "nullCount": 0, - "nullProportion": 0.0, - "min": "1", - "max": "3" - }, - { - "fieldPath": "trip_date", - "nullCount": 0, - "nullProportion": 0.0, - "min": "2000-01-01T12:00:00+00:00", - "max": "2000-01-04T12:00:00+00:00" - }, - { - "fieldPath": "trip_id", - "nullCount": 0, - "nullProportion": 0.0, - "min": "1000371", - "max": "1000375" - }, - { - "fieldPath": "trip_distance", - "nullCount": 0, - "nullProportion": 0.0, - "min": "0.0", - "max": "8.399999618530273" - }, - { - "fieldPath": "fare_amount", - "nullCount": 0, - "nullProportion": 0.0, - "min": "0.0", - "max": "42.13" - }, - { - "fieldPath": "store_and_fwd_flag", - "nullCount": 0, - "nullProportion": 0.0 - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "iceberg-test" + { + "fieldPath": "trip_date", + "nullCount": 0, + "nullProportion": 0.0, + "min": "2000-01-01T12:00:00+00:00", + "max": "2000-01-04T12:00:00+00:00" + }, + { + "fieldPath": "trip_id", + "nullCount": 0, + "nullProportion": 0.0, + "min": "1000371", + "max": "1000375" + }, + { + "fieldPath": "trip_distance", + "nullCount": 0, + "nullProportion": 0.0, + "min": "0.0", + "max": "8.399999618530273" + }, + { + "fieldPath": "fare_amount", + "nullCount": 0, + "nullProportion": 0.0, + "min": "0.0", + "max": "42.13" + }, + { + "fieldPath": "store_and_fwd_flag", + "nullCount": 0, + "nullProportion": 0.0 + } + ] } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "iceberg-test", + "lastRunId": "no-run-id-provided" } - ] \ No newline at end of file +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/iceberg/iceberg_profile_to_file.yml b/metadata-ingestion/tests/integration/iceberg/iceberg_profile_to_file.yml index 197c03bf2ee8d..38c216e1bf49f 100644 --- a/metadata-ingestion/tests/integration/iceberg/iceberg_profile_to_file.yml +++ b/metadata-ingestion/tests/integration/iceberg/iceberg_profile_to_file.yml @@ -4,9 +4,8 @@ source: type: iceberg config: catalog: - name: default - type: rest - config: + default: + type: rest uri: http://localhost:8181 s3.access-key-id: admin s3.secret-access-key: password diff --git a/metadata-ingestion/tests/integration/iceberg/iceberg_to_file.yml b/metadata-ingestion/tests/integration/iceberg/iceberg_to_file.yml index 8b5d035aed259..2624ee6d0df81 100644 --- a/metadata-ingestion/tests/integration/iceberg/iceberg_to_file.yml +++ b/metadata-ingestion/tests/integration/iceberg/iceberg_to_file.yml @@ -4,9 +4,8 @@ source: type: iceberg config: catalog: - name: default - type: rest - config: + default: + type: rest uri: http://localhost:8181 s3.access-key-id: admin s3.secret-access-key: password diff --git a/metadata-ingestion/tests/integration/iceberg/test_iceberg.py b/metadata-ingestion/tests/integration/iceberg/test_iceberg.py index 24a636077bfdd..5a12afa457f01 100644 --- a/metadata-ingestion/tests/integration/iceberg/test_iceberg.py +++ b/metadata-ingestion/tests/integration/iceberg/test_iceberg.py @@ -79,9 +79,8 @@ def test_iceberg_stateful_ingest( "type": "iceberg", "config": { "catalog": { - "name": "default", - "type": "rest", - "config": { + "default": { + "type": "rest", "uri": "http://localhost:8181", "s3.access-key-id": "admin", "s3.secret-access-key": "password", diff --git a/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml b/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml index 260887ff3d1a0..85b6a310a06d2 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml +++ b/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml @@ -1,5 +1,4 @@ --- -version: '3.8' services: connect: image: confluentinc/cp-kafka-connect:7.4.0 @@ -32,7 +31,7 @@ services: # confluent-hub install --no-prompt debezium/debezium-connector-mysql:1.7.0 # - #confluent-hub install --no-prompt wepay/kafka-connect-bigquery:1.6.8 + confluent-hub install --no-prompt wepay/kafka-connect-bigquery:1.6.8 # confluent-hub install --no-prompt mongodb/kafka-connect-mongodb:1.10.1 # diff --git a/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_bigquery_sink_mces_golden.json b/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_bigquery_sink_mces_golden.json new file mode 100644 index 0000000000000..086dfbdd67b1c --- /dev/null +++ b/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_bigquery_sink_mces_golden.json @@ -0,0 +1,48 @@ +[ +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(kafka-connect,connect-instance-1.bigquery-sink-connector,PROD)", + "changeType": "UPSERT", + "aspectName": "dataFlowInfo", + "aspect": { + "json": { + "customProperties": { + "connector.class": "com.wepay.kafka.connect.bigquery.BigQuerySinkConnector", + "autoCreateTables": "true", + "transforms.TableNameTransformation.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.TableNameTransformation.replacement": "my_dest_table_name", + "topics": "kafka-topic-name", + "transforms.TableNameTransformation.regex": ".*", + "transforms": "TableNameTransformation", + "name": "bigquery-sink-connector", + "project": "my-gcp-project", + "datasets": "kafka-topic-name=mybqdataset", + "defaultDataset": "mybqdataset" + }, + "name": "bigquery-sink-connector", + "description": "Sink connector using `com.wepay.kafka.connect.bigquery.BigQuerySinkConnector` plugin." + } + }, + "systemMetadata": { + "lastObserved": 1635166800000, + "runId": "kafka-connect-run", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(kafka-connect,connect-instance-1.bigquery-sink-connector,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1635166800000, + "runId": "kafka-connect-run", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_bigquery_sink_to_file.yml b/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_bigquery_sink_to_file.yml new file mode 100644 index 0000000000000..0df332acb32de --- /dev/null +++ b/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_bigquery_sink_to_file.yml @@ -0,0 +1,15 @@ +--- +run_id: kafka-connect-run + +source: + type: kafka-connect + config: + platform_instance: connect-instance-1 + connect_uri: http://localhost:28083 + connector_patterns: + allow: + - bigquery-sink-connector +sink: + type: file + config: + filename: "./kafka_connect_mces.json" diff --git a/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_to_file.yml b/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_to_file.yml index 4946cae8c4859..4adb57fba6814 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_to_file.yml +++ b/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_to_file.yml @@ -10,6 +10,7 @@ source: deny: - source_mongodb_connector - confluent_s3_sink_connector + - bigquery-sink-connector provided_configs: - provider: env path_key: MYSQL_PORT diff --git a/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py b/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py index 26f3d50c1167b..d0f4fc35fc03e 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py +++ b/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py @@ -333,6 +333,30 @@ def loaded_kafka_connect(kafka_connect_runner): r.raise_for_status() assert r.status_code == 201 + # Creating BigQuery sink connector + r = requests.post( + KAFKA_CONNECT_ENDPOINT, + headers={"Content-Type": "application/json"}, + data="""{ + "name": "bigquery-sink-connector", + "config": { + "connector.class": "com.wepay.kafka.connect.bigquery.BigQuerySinkConnector", + "autoCreateTables": "true", + "transforms.TableNameTransformation.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.TableNameTransformation.replacement": "my_dest_table_name", + "topics": "kafka-topic-name", + "transforms.TableNameTransformation.regex": ".*", + "transforms": "TableNameTransformation", + "name": "bigquery-sink-connector", + "project": "my-gcp-project", + "defaultDataset": "mybqdataset", + "datasets": "kafka-topic-name=mybqdataset" + } + } + """, + ) + assert r.status_code == 201 # Created + # Give time for connectors to process the table data kafka_connect_runner.wait_until_responsive( timeout=30, @@ -637,3 +661,22 @@ def test_kafka_connect_snowflake_sink_ingest( output_path=tmp_path / "kafka_connect_snowflake_sink_mces.json", golden_path=f"{test_resources_dir}/{golden_file}", ) + + +@freeze_time(FROZEN_TIME) +def test_kafka_connect_bigquery_sink_ingest( + loaded_kafka_connect, pytestconfig, tmp_path, test_resources_dir +): + # Run the metadata ingestion pipeline. + config_file = ( + test_resources_dir / "kafka_connect_bigquery_sink_to_file.yml" + ).resolve() + run_datahub_cmd(["ingest", "-c", f"{config_file}"], tmp_path=tmp_path) + + # Verify the output. + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / "kafka_connect_mces.json", + golden_path=test_resources_dir / "kafka_connect_bigquery_sink_mces_golden.json", + ignore_paths=[], + ) diff --git a/metadata-ingestion/tests/integration/kafka/docker-compose.yml b/metadata-ingestion/tests/integration/kafka/docker-compose.yml index 0a4422e07515c..044842850f36a 100644 --- a/metadata-ingestion/tests/integration/kafka/docker-compose.yml +++ b/metadata-ingestion/tests/integration/kafka/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3.8" services: zookeeper: image: confluentinc/cp-zookeeper:7.2.2 diff --git a/metadata-ingestion/tests/integration/kafka/kafka_mces_golden.json b/metadata-ingestion/tests/integration/kafka/kafka_mces_golden.json index 7dd328168e84c..7df790b74e829 100644 --- a/metadata-ingestion/tests/integration/kafka/kafka_mces_golden.json +++ b/metadata-ingestion/tests/integration/kafka/kafka_mces_golden.json @@ -270,7 +270,8 @@ "retention.ms": "604800000", "cleanup.policy": "delete", "max.message.bytes": "1048588", - "unclean.leader.election.enable": "false" + "unclean.leader.election.enable": "false", + "Schema Name": "key_value_topic-value" }, "name": "key_value_topic", "description": "Value schema for kafka topic", @@ -472,7 +473,8 @@ "retention.ms": "604800000", "cleanup.policy": "delete", "max.message.bytes": "1048588", - "unclean.leader.election.enable": "false" + "unclean.leader.election.enable": "false", + "Schema Name": "value_topic-value" }, "name": "value_topic", "description": "Value schema for kafka topic", @@ -522,6 +524,889 @@ "lastRunId": "no-run-id-provided" } }, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic-key,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "key_topic-key", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "5e468f7aa532c2f2ed9686ff3ec943ec", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"UserKey\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Key schema for kafka topic\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"namespace\",\"type\":\"string\"}]}", + "documentSchemaType": "AVRO", + "keySchema": "{\"type\":\"record\",\"name\":\"UserKey\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Key schema for kafka topic\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"namespace\",\"type\":\"string\"}]}", + "keySchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=long].id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "id", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=string].namespace", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "namespace", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "[version=2.0].[type=UserKey].[type=long].id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "id", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=UserKey].[type=string].namespace", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "namespace", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/kafka" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "key_topic-key", + "description": "Key schema for kafka topic", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic-key,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic-key,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic-key,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "key_value_topic-key", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "5e468f7aa532c2f2ed9686ff3ec943ec", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"UserKey\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Key schema for kafka topic\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"namespace\",\"type\":\"string\"}]}", + "documentSchemaType": "AVRO", + "keySchema": "{\"type\":\"record\",\"name\":\"UserKey\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Key schema for kafka topic\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"namespace\",\"type\":\"string\"}]}", + "keySchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=long].id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "id", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=string].namespace", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "namespace", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "[version=2.0].[type=UserKey].[type=long].id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "id", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=UserKey].[type=string].namespace", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "namespace", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/kafka" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "key_value_topic-key", + "description": "Key schema for kafka topic", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic-key,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic-key,PROD)", + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "json": { + "domains": [ + "urn:li:domain:sales" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic-key,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic-value,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "key_value_topic-value", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "c9b692583e304b9cb703ffa748a9f37d", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"CreateUserRequest\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Value schema for kafka topic\",\"fields\":[{\"name\":\"email\",\"type\":\"string\",\"tags\":[\"Email\"]},{\"name\":\"firstName\",\"type\":\"string\",\"tags\":[\"Name\"]},{\"name\":\"lastName\",\"type\":\"string\",\"tags\":[\"Name\"]}],\"tags\":[\"PII\"]}", + "documentSchemaType": "AVRO", + "keySchema": "{\"type\":\"record\",\"name\":\"CreateUserRequest\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Value schema for kafka topic\",\"fields\":[{\"name\":\"email\",\"type\":\"string\",\"tags\":[\"Email\"]},{\"name\":\"firstName\",\"type\":\"string\",\"tags\":[\"Name\"]},{\"name\":\"lastName\",\"type\":\"string\",\"tags\":[\"Name\"]}],\"tags\":[\"PII\"]}", + "keySchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[key=True].[type=CreateUserRequest].[type=string].email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "email", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Email" + } + ] + }, + "isPartOfKey": true, + "jsonProps": "{\"tags\": [\"Email\"]}" + }, + { + "fieldPath": "[version=2.0].[key=True].[type=CreateUserRequest].[type=string].firstName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "firstName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": true, + "jsonProps": "{\"tags\": [\"Name\"]}" + }, + { + "fieldPath": "[version=2.0].[key=True].[type=CreateUserRequest].[type=string].lastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "lastName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": true, + "jsonProps": "{\"tags\": [\"Name\"]}" + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "email", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Email" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Email\"]}" + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].firstName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "firstName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Name\"]}" + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].lastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "lastName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Name\"]}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/kafka" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:PII" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "key_value_topic-value", + "description": "Value schema for kafka topic", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic-value,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic-value,PROD)", + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "json": { + "domains": [ + "urn:li:domain:sales" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic-value,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic-key,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "value_topic-key", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "c088cd2eb2de57e32c00b32d4871ec72", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "\"string\"", + "documentSchemaType": "AVRO", + "keySchema": "\"string\"", + "keySchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[key=True].[type=string]", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "[version=2.0].[type=string]", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/kafka" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "value_topic-key", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic-key,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic-key,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic-value,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "value_topic-value", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "c9b692583e304b9cb703ffa748a9f37d", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"CreateUserRequest\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Value schema for kafka topic\",\"fields\":[{\"name\":\"email\",\"type\":\"string\",\"tags\":[\"Email\"]},{\"name\":\"firstName\",\"type\":\"string\",\"tags\":[\"Name\"]},{\"name\":\"lastName\",\"type\":\"string\",\"tags\":[\"Name\"]}],\"tags\":[\"PII\"]}", + "documentSchemaType": "AVRO", + "keySchema": "{\"type\":\"record\",\"name\":\"CreateUserRequest\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Value schema for kafka topic\",\"fields\":[{\"name\":\"email\",\"type\":\"string\",\"tags\":[\"Email\"]},{\"name\":\"firstName\",\"type\":\"string\",\"tags\":[\"Name\"]},{\"name\":\"lastName\",\"type\":\"string\",\"tags\":[\"Name\"]}],\"tags\":[\"PII\"]}", + "keySchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[key=True].[type=CreateUserRequest].[type=string].email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "email", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Email" + } + ] + }, + "isPartOfKey": true, + "jsonProps": "{\"tags\": [\"Email\"]}" + }, + { + "fieldPath": "[version=2.0].[key=True].[type=CreateUserRequest].[type=string].firstName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "firstName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": true, + "jsonProps": "{\"tags\": [\"Name\"]}" + }, + { + "fieldPath": "[version=2.0].[key=True].[type=CreateUserRequest].[type=string].lastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "lastName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": true, + "jsonProps": "{\"tags\": [\"Name\"]}" + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "email", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Email" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Email\"]}" + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].firstName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "firstName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Name\"]}" + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].lastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "lastName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Name\"]}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/kafka" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:PII" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "value_topic-value", + "description": "Value schema for kafka topic", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic-value,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic-value,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:Email", diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json index 90b3f0119fd48..fa1a5eebe3e5b 100644 --- a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json +++ b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json @@ -6,7 +6,6 @@ "aspects": [ { "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { - "email": "simpons-group", "admins": [], "members": [ "urn:li:corpuser:hsimpson", @@ -20,7 +19,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -35,7 +35,6 @@ }, "active": true, "displayName": "Bart Simpson", - "email": "bsimpson", "title": "Mr. Boss", "firstName": "Bart", "lastName": "Simpson", @@ -52,7 +51,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -87,7 +87,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -102,7 +103,6 @@ }, "active": true, "displayName": "Lisa Simpson", - "email": "lsimpson", "firstName": "Lisa", "lastName": "Simpson", "fullName": "Lisa Simpson" @@ -118,7 +118,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -133,7 +134,6 @@ }, "active": true, "displayName": "Maggie Simpson", - "email": "msimpson", "firstName": "Maggie", "lastName": "Simpson", "fullName": "Maggie Simpson" @@ -149,7 +149,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -164,7 +165,6 @@ }, "active": true, "displayName": "Hester Bevan", - "email": "hbevan", "firstName": "Hester", "lastName": "Bevan", "fullName": "Hester Bevan" @@ -180,7 +180,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -195,7 +196,6 @@ }, "active": true, "displayName": "Evalyn Haas", - "email": "ehaas", "firstName": "Evalyn", "lastName": "Haas", "fullName": "Evalyn Haas" @@ -211,7 +211,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -221,7 +222,6 @@ "aspects": [ { "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { - "email": "HR Department", "admins": [], "members": [], "groups": [] @@ -232,7 +232,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -242,7 +243,6 @@ "aspects": [ { "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { - "email": "Finance Department", "admins": [], "members": [], "groups": [] @@ -253,7 +253,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -268,7 +269,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -283,7 +285,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -298,7 +301,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -313,7 +317,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -328,7 +333,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -343,7 +349,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -358,7 +365,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -373,7 +381,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -388,7 +397,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_group_stateful.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_group_stateful.json index d4bafbd664a15..4abeec224c0d9 100644 --- a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_group_stateful.json +++ b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_group_stateful.json @@ -6,7 +6,6 @@ "aspects": [ { "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { - "email": "HR Department", "admins": [], "members": [], "groups": [] @@ -17,7 +16,9 @@ }, "systemMetadata": { "lastObserved": 1660460400000, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "ldap-test-pipeline" } }, { @@ -26,12 +27,15 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1660460400000, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "ldap-test-pipeline" } }, { @@ -40,12 +44,15 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": true}", - "contentType": "application/json" + "json": { + "removed": true + } }, "systemMetadata": { "lastObserved": 1660460400000, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "ldap-test-pipeline" } } -] +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_stateful.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_stateful.json index 7462ea1458278..b72e1da740be4 100644 --- a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_stateful.json +++ b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_stateful.json @@ -9,7 +9,6 @@ "customProperties": {}, "active": true, "displayName": "Bart Simpson", - "email": "bsimpson", "title": "Mr. Boss", "firstName": "Bart", "lastName": "Simpson", @@ -26,7 +25,9 @@ }, "systemMetadata": { "lastObserved": 1660460400000, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "ldap-test-pipeline" } }, { @@ -41,7 +42,9 @@ }, "systemMetadata": { "lastObserved": 1660460400000, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "ldap-test-pipeline" } }, { @@ -56,7 +59,9 @@ }, "systemMetadata": { "lastObserved": 1660460400000, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "ldap-test-pipeline" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_group_stateful.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_group_stateful.json index dc97146982795..c4f8e20662bec 100644 --- a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_group_stateful.json +++ b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_group_stateful.json @@ -6,7 +6,6 @@ "aspects": [ { "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { - "email": "HR Department", "admins": [], "members": [], "groups": [] @@ -17,7 +16,9 @@ }, "systemMetadata": { "lastObserved": 1660460400000, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "ldap-test-pipeline" } }, { @@ -27,7 +28,6 @@ "aspects": [ { "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { - "email": "Finance Department", "admins": [], "members": [], "groups": [] @@ -38,7 +38,9 @@ }, "systemMetadata": { "lastObserved": 1660460400000, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "ldap-test-pipeline" } }, { @@ -47,12 +49,15 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1660460400000, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "ldap-test-pipeline" } }, { @@ -61,12 +66,15 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1660460400000, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "ldap-test-pipeline" } }, { @@ -75,12 +83,15 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": true}", - "contentType": "application/json" + "json": { + "removed": true + } }, "systemMetadata": { "lastObserved": 1660460400000, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "ldap-test-pipeline" } } -] +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_stateful.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_stateful.json index d79bb56f1ab5f..2e7dd6710fe4f 100644 --- a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_stateful.json +++ b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_stateful.json @@ -9,7 +9,6 @@ "customProperties": {}, "active": true, "displayName": "Bart Simpson", - "email": "bsimpson", "title": "Mr. Boss", "firstName": "Bart", "lastName": "Simpson", @@ -26,7 +25,9 @@ }, "systemMetadata": { "lastObserved": 1660460400000, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "ldap-test-pipeline" } }, { @@ -59,7 +60,9 @@ }, "systemMetadata": { "lastObserved": 1660460400000, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "ldap-test-pipeline" } }, { @@ -74,7 +77,9 @@ }, "systemMetadata": { "lastObserved": 1660460400000, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "ldap-test-pipeline" } }, { @@ -89,7 +94,9 @@ }, "systemMetadata": { "lastObserved": 1660460400000, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "ldap-test-pipeline" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/ldap/ldap_memberof_mces_golden.json b/metadata-ingestion/tests/integration/ldap/ldap_memberof_mces_golden.json index 016cbe49d7741..445c86b27222d 100644 --- a/metadata-ingestion/tests/integration/ldap/ldap_memberof_mces_golden.json +++ b/metadata-ingestion/tests/integration/ldap/ldap_memberof_mces_golden.json @@ -9,7 +9,6 @@ "customProperties": {}, "active": true, "displayName": "Hester Bevan", - "email": "hbevan", "firstName": "Hester", "lastName": "Bevan", "fullName": "Hester Bevan" @@ -28,7 +27,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -41,7 +41,6 @@ "customProperties": {}, "active": true, "displayName": "Evalyn Haas", - "email": "ehaas", "firstName": "Evalyn", "lastName": "Haas", "fullName": "Evalyn Haas" @@ -60,7 +59,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -75,7 +75,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } }, { @@ -90,7 +91,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test" + "runId": "ldap-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/looker/golden_looker_mces.json b/metadata-ingestion/tests/integration/looker/golden_looker_mces.json index 2bebb247a41aa..1ce1b4b4750da 100644 --- a/metadata-ingestion/tests/integration/looker/golden_looker_mces.json +++ b/metadata-ingestion/tests/integration/looker/golden_looker_mces.json @@ -35,7 +35,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -51,7 +52,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -76,9 +78,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, @@ -112,7 +111,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -137,9 +137,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } } @@ -149,7 +146,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -174,9 +172,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, @@ -218,9 +213,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } } @@ -230,7 +222,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -251,7 +244,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -267,7 +261,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -283,7 +278,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -301,7 +297,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -321,7 +318,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -342,7 +340,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -358,7 +357,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -374,7 +374,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -392,7 +393,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -412,7 +414,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -509,7 +512,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -527,7 +531,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -543,7 +548,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -559,7 +565,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -583,7 +590,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -680,7 +688,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -698,7 +707,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -714,7 +724,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -730,7 +741,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -754,7 +766,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -774,7 +787,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -794,7 +808,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -814,7 +829,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -830,7 +846,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -846,7 +863,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -862,7 +880,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -878,7 +897,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -894,7 +914,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json index 875d5a7356091..87430ef1067ff 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json @@ -76,9 +76,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, @@ -137,9 +134,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json b/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json index 3ba4f1fbb5da9..3f11798d0aa5a 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json @@ -312,9 +312,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, @@ -373,9 +370,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json index be49879f4e263..ec2c46c5daf27 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json @@ -312,9 +312,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, @@ -373,9 +370,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json index f5cac9cb2d025..bb3c3ccb4e214 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json @@ -17,7 +17,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -33,7 +34,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -49,7 +51,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -67,7 +70,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -87,7 +91,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -138,7 +143,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -156,7 +162,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -183,7 +190,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -231,7 +239,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -247,7 +256,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -263,7 +273,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -287,7 +298,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -312,9 +324,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, @@ -348,7 +357,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -373,9 +383,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, @@ -409,7 +416,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -460,7 +468,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -478,7 +487,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -494,7 +504,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -510,7 +521,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -526,7 +538,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -550,7 +563,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -571,7 +585,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -587,7 +602,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -603,7 +619,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -621,7 +638,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -641,7 +659,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -662,7 +681,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -678,7 +698,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -694,7 +715,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -712,7 +734,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -732,7 +755,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -829,7 +853,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -847,7 +872,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -863,7 +889,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -879,7 +906,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -903,7 +931,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -1000,7 +1029,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -1018,7 +1048,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -1034,7 +1065,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -1050,7 +1082,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -1074,7 +1107,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -1094,7 +1128,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -1114,7 +1149,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -1134,7 +1170,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -1150,7 +1187,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -1166,7 +1204,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } }, { @@ -1182,7 +1221,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "execution-1" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json index ab1c671960982..a7b8abed02da3 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json @@ -312,9 +312,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, @@ -373,9 +370,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json index 0f55e26a2baab..a1a7747c741a6 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json @@ -312,9 +312,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, @@ -373,9 +370,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json index fcad66ce61f23..4d8e2e79eafad 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json @@ -76,9 +76,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, @@ -137,9 +134,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json b/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json index 48eddea1e2e00..e3cbf43059c7c 100644 --- a/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json +++ b/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json @@ -17,7 +17,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -33,7 +34,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -49,7 +51,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -67,7 +70,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -87,7 +91,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -138,7 +143,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -156,7 +162,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -183,7 +190,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -231,7 +239,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -247,7 +256,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -263,7 +273,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -287,7 +298,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -312,9 +324,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, @@ -348,7 +357,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -373,9 +383,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } }, @@ -409,7 +416,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -430,7 +438,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -446,7 +455,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -462,7 +472,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -480,7 +491,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -500,7 +512,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -597,7 +610,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -615,7 +629,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -631,7 +646,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -647,7 +663,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -671,7 +688,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -691,7 +709,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -711,7 +730,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -731,7 +751,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -747,7 +768,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -763,7 +785,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -779,12 +802,13 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7", + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.10)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -795,7 +819,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -811,12 +836,13 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(looker,dashboard_elements.10)", + "entityType": "container", + "entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -827,7 +853,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } }, { @@ -843,7 +870,8 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "looker-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "stateful-looker-pipeline" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json index e94089d9b4ea4..f3da2dbfac2f0 100644 --- a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json +++ b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json @@ -76,9 +76,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } } @@ -113,9 +110,6 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } } diff --git a/metadata-ingestion/tests/integration/looker/test_looker.py b/metadata-ingestion/tests/integration/looker/test_looker.py index 3e049f8b2ef4e..e1cedee33dcb6 100644 --- a/metadata-ingestion/tests/integration/looker/test_looker.py +++ b/metadata-ingestion/tests/integration/looker/test_looker.py @@ -1050,23 +1050,34 @@ def test_upstream_cll(pytestconfig, tmp_path, mock_time, mock_datahub_graph): ), ], ) + config = mock.MagicMock() + + config.view_naming_pattern.replace_variables.return_value = "dataset_lineages" + config.platform_name = "snowflake" + config.platform_instance = "sales" + config.env = "DEV" looker_explore: Optional[LookerExplore] = looker_common.LookerExplore.from_api( model="fake", explore_name="my_explore_name", client=mocked_client, reporter=mock.MagicMock(), - source_config=mock.MagicMock(), + source_config=config, ) assert looker_explore is not None assert looker_explore.name == "my_explore_name" assert looker_explore.fields is not None assert len(looker_explore.fields) == 3 + assert ( - looker_explore.fields[2].upstream_fields[0] == "dataset_lineages.createdon" + looker_explore.fields[2].upstream_fields[0].table + == "urn:li:dataset:(urn:li:dataPlatform:snowflake," + "sales.dataset_lineages,DEV)" ) + assert looker_explore.fields[2].upstream_fields[0].column == "createdon" + @freeze_time(FROZEN_TIME) def test_explore_tags(pytestconfig, tmp_path, mock_time, mock_datahub_graph): diff --git a/metadata-ingestion/tests/integration/lookml/duplicate_field_ingestion_golden.json b/metadata-ingestion/tests/integration/lookml/duplicate_field_ingestion_golden.json index b06b59ba43654..ca9a1503a6854 100644 --- a/metadata-ingestion/tests/integration/lookml/duplicate_field_ingestion_golden.json +++ b/metadata-ingestion/tests/integration/lookml/duplicate_field_ingestion_golden.json @@ -116,7 +116,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "# File was added to check duplicate field issue\n\nview: dataset_lineages {\n sql_table_name: \"PUBLIC\".\"DATASET_LINEAGES\"\n ;;\n\n dimension: createdon {\n type: date\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension_group: createdon {\n type: time\n timeframes: [\n raw,\n time,\n date,\n week,\n month,\n quarter,\n year\n ]\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension: entity {\n type: string\n sql: ${TABLE}.\"ENTITY\" ;;\n }\n\n dimension: metadata {\n type: string\n sql: ${TABLE}.\"METADATA\" ;;\n }\n\n dimension: urn {\n type: string\n sql: ${TABLE}.\"URN\" ;;\n }\n\n dimension: version {\n type: number\n sql: ${TABLE}.\"VERSION\" ;;\n }\n\n measure: count {\n type: count\n drill_fields: []\n }\n}\n", + "viewLogic": "# File was added to check duplicate field issue\n\nview: dataset_lineages {\n sql_table_name: \"PUBLIC\".\"DATASET_LINEAGES\"\n ;;\n\n dimension: createdon {\n type: date\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension_group: createdon {\n type: time\n timeframes: [\n raw,\n time,\n date,\n week,\n month,\n quarter,\n year\n ]\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension: entity {\n type: string\n sql: ${TABLE}.\"ENTITY\" ;;\n }\n\n dimension: metadata {\n type: string\n sql: ${TABLE}.\"METADATA\" ;;\n }\n\n dimension: urn {\n type: string\n sql: ${TABLE}.\"URN\" ;;\n }\n\n dimension: version {\n type: number\n tags: [\"softVersion\"]\n sql: ${TABLE}.\"VERSION\" ;;\n }\n\n measure: count {\n type: count\n drill_fields: []\n }\n}\n", "viewLanguage": "lookml" } }, @@ -167,7 +167,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,.public.dataset_lineages,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.public.dataset_lineages,PROD)", "type": "VIEW" } ], @@ -175,7 +175,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.public.dataset_lineages,PROD),entity)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.dataset_lineages,PROD),entity)" ], "downstreamType": "FIELD", "downstreams": [ @@ -186,7 +186,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.public.dataset_lineages,PROD),metadata)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.dataset_lineages,PROD),metadata)" ], "downstreamType": "FIELD", "downstreams": [ @@ -197,7 +197,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.public.dataset_lineages,PROD),urn)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.dataset_lineages,PROD),urn)" ], "downstreamType": "FIELD", "downstreams": [ @@ -208,7 +208,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.public.dataset_lineages,PROD),version)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.dataset_lineages,PROD),version)" ], "downstreamType": "FIELD", "downstreams": [ @@ -219,7 +219,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.public.dataset_lineages,PROD),createdon)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.dataset_lineages,PROD),createdon)" ], "downstreamType": "FIELD", "downstreams": [ @@ -230,7 +230,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.public.dataset_lineages,PROD),count)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.dataset_lineages,PROD),count)" ], "downstreamType": "FIELD", "downstreams": [ @@ -338,6 +338,9 @@ "recursive": false, "globalTags": { "tags": [ + { + "tag": "urn:li:tag:softVersion" + }, { "tag": "urn:li:tag:Dimension" } @@ -484,5 +487,21 @@ "runId": "lookml-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:softVersion", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "softVersion" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/expected_output.json b/metadata-ingestion/tests/integration/lookml/expected_output.json index 931830eecf0c3..d870c6dee4065 100644 --- a/metadata-ingestion/tests/integration/lookml/expected_output.json +++ b/metadata-ingestion/tests/integration/lookml/expected_output.json @@ -167,9 +167,66 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,..my_table,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,my_table,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,my_table,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,my_table,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,my_table,PROD),is_latest)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),is_latest)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,my_table,PROD),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,my_table,PROD),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -428,6 +485,52 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -662,7 +765,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,.looker_schema.include_able,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.looker_schema.include_able,PROD)", "type": "VIEW" } ] @@ -788,7 +891,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,.looker_schema.events,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.looker_schema.events,PROD)", "type": "VIEW" } ] @@ -914,7 +1017,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,.looker_schema.events,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.looker_schema.events,PROD)", "type": "VIEW" } ], @@ -922,7 +1025,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.looker_schema.events,PROD),additional_measure)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.looker_schema.events,PROD),additional_measure)" ], "downstreamType": "FIELD", "downstreams": [ @@ -1098,7 +1201,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,..autodetect_sql_name_based_on_view_name,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,..autodetect_sql_name_based_on_view_name,PROD)", "type": "VIEW" } ] @@ -1224,7 +1327,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,.looker_schema.include_able,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.looker_schema.include_able,PROD)", "type": "VIEW" } ] @@ -1299,7 +1402,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "date DATE encode ZSTD, \n platform VARCHAR(20) encode ZSTD AS aliased_platform, \n country VARCHAR(20) encode ZSTD", + "viewLogic": "SELECT date AS DATE,\n platform AS aliased_platform,\n country", "viewLanguage": "sql" } }, @@ -1350,9 +1453,44 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,..fragment_derived_view,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,fragment_derived_view,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,fragment_derived_view,PROD),date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,fragment_derived_view,PROD),platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),aliased_platform)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,fragment_derived_view,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -1377,7 +1515,7 @@ }, "fields": [ { - "fieldPath": "aliased_platform", + "fieldPath": "date", "nullable": false, "description": "", "label": "", @@ -1388,13 +1526,10 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false }, { - "fieldPath": "country", + "fieldPath": "aliased_platform", "nullable": false, "description": "", "label": "", @@ -1405,13 +1540,10 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false }, { - "fieldPath": "date", + "fieldPath": "country", "nullable": false, "description": "", "label": "", @@ -1422,9 +1554,6 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } ], @@ -1503,7 +1632,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", + "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n order.region='ap-south-1'\n GROUP BY 1", "viewLanguage": "sql" } }, @@ -1554,12 +1683,88 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,..order,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,order,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,order,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,order,PROD),sale_price)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),lifetime_spend)" + ], + "confidenceScore": 1.0 + } ] } }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "customer_facts", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lifetime_spend", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { @@ -1680,7 +1885,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,.ecommerce.ability,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.ecommerce.ability,PROD)", "type": "VIEW" } ], @@ -1688,7 +1893,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.ecommerce.ability,PROD),pk)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.ecommerce.ability,PROD),pk)" ], "downstreamType": "FIELD", "downstreams": [ @@ -1699,7 +1904,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.ecommerce.ability,PROD),count)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.ecommerce.ability,PROD),count)" ], "downstreamType": "FIELD", "downstreams": [ @@ -1896,7 +2101,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,..owners,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,..owners,PROD)", "type": "VIEW" } ], @@ -1904,7 +2109,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,..owners,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,..owners,PROD),id)" ], "downstreamType": "FIELD", "downstreams": [ @@ -1915,7 +2120,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,..owners,PROD),owner_name)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,..owners,PROD),owner_name)" ], "downstreamType": "FIELD", "downstreams": [ @@ -2394,7 +2599,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,.flightstats.accidents,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.flightstats.accidents,PROD)", "type": "VIEW" } ], @@ -2402,7 +2607,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.flightstats.accidents,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.flightstats.accidents,PROD),id)" ], "downstreamType": "FIELD", "downstreams": [ diff --git a/metadata-ingestion/tests/integration/lookml/field_tag_ingestion_golden.json b/metadata-ingestion/tests/integration/lookml/field_tag_ingestion_golden.json new file mode 100644 index 0000000000000..19352a85249ba --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/field_tag_ingestion_golden.json @@ -0,0 +1,567 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "looker", + "env": "PROD", + "project_name": "lkml_samples" + }, + "name": "lkml_samples" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:looker" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "LookML Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "# File was added to check duplicate field issue\n\nview: dataset_lineages {\n sql_table_name: \"PUBLIC\".\"DATASET_LINEAGES\"\n ;;\n\n dimension: createdon {\n type: date\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension_group: createdon {\n type: time\n timeframes: [\n raw,\n time,\n date,\n week,\n month,\n quarter,\n year\n ]\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension: entity {\n type: string\n sql: ${TABLE}.\"ENTITY\" ;;\n }\n\n dimension: metadata {\n type: string\n sql: ${TABLE}.\"METADATA\" ;;\n }\n\n dimension: urn {\n type: string\n sql: ${TABLE}.\"URN\" ;;\n }\n\n dimension: version {\n type: number\n tags: [\"softVersion\"]\n sql: ${TABLE}.\"VERSION\" ;;\n }\n\n measure: count {\n type: count\n drill_fields: []\n }\n}\n", + "viewLanguage": "lookml" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Develop/lkml_samples/" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.public.dataset_lineages,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.dataset_lineages,PROD),entity)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD),entity)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.dataset_lineages,PROD),metadata)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD),metadata)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.dataset_lineages,PROD),urn)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD),urn)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.dataset_lineages,PROD),version)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD),version)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.dataset_lineages,PROD),createdon)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD),createdon)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.dataset_lineages,PROD),count)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD),count)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "dataset_lineages", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "entity", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "metadata", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "urn", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "version", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:softVersion" + }, + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "createdon", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "time", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + }, + { + "tag": "urn:li:tag:Temporal" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "count", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "count", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Measure" + } + ] + }, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "looker.file.path": "dataset_lineages.view.lkml", + "looker.model": "data" + }, + "name": "dataset_lineages", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.dataset_lineages,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Dimension", + "aspects": [ + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Dimension", + "description": "A tag that is applied to all dimension fields." + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Temporal", + "aspects": [ + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Temporal", + "description": "A tag that is applied to all time-based (temporal) fields such as timestamps or durations." + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Measure", + "aspects": [ + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Measure", + "description": "A tag that is applied to all measures (metrics). Measures are typically the columns that you aggregate on" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Dimension", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Measure", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Temporal", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:softVersion", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "softVersion" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/lkml_samples/liquid.view.lkml b/metadata-ingestion/tests/integration/lookml/lkml_samples/liquid.view.lkml index bfeebfba09aa6..8d26db7931b0d 100644 --- a/metadata-ingestion/tests/integration/lookml/lkml_samples/liquid.view.lkml +++ b/metadata-ingestion/tests/integration/lookml/lkml_samples/liquid.view.lkml @@ -1,14 +1,14 @@ view: customer_facts { derived_table: { -sql: - SELECT - customer_id, - SUM(sale_price) AS lifetime_spend - FROM - order - WHERE - {% condition order_region %} order.region {% endcondition %} - GROUP BY 1 - ;; - } + sql: + SELECT + customer_id, + SUM(sale_price) AS lifetime_spend + FROM + order + WHERE + {% condition order_region %} order.region {% endcondition %} + GROUP BY 1 + ;; + } } diff --git a/metadata-ingestion/tests/integration/lookml/lkml_samples/nested/fragment_derived.view.lkml b/metadata-ingestion/tests/integration/lookml/lkml_samples/nested/fragment_derived.view.lkml index 284bf4172cfdc..d8397cc355e0e 100644 --- a/metadata-ingestion/tests/integration/lookml/lkml_samples/nested/fragment_derived.view.lkml +++ b/metadata-ingestion/tests/integration/lookml/lkml_samples/nested/fragment_derived.view.lkml @@ -1,9 +1,9 @@ view: fragment_derived_view { derived_table: { - sql: date DATE encode ZSTD, - platform VARCHAR(20) encode ZSTD AS aliased_platform, - country VARCHAR(20) encode ZSTD + sql: SELECT date AS DATE, + platform AS aliased_platform, + country ;; } } diff --git a/metadata-ingestion/tests/integration/lookml/lkml_samples_duplicate_field/dataset_lineages.view.lkml b/metadata-ingestion/tests/integration/lookml/lkml_samples_duplicate_field/dataset_lineages.view.lkml index 6062993f320d3..c29c47719810a 100644 --- a/metadata-ingestion/tests/integration/lookml/lkml_samples_duplicate_field/dataset_lineages.view.lkml +++ b/metadata-ingestion/tests/integration/lookml/lkml_samples_duplicate_field/dataset_lineages.view.lkml @@ -40,6 +40,7 @@ view: dataset_lineages { dimension: version { type: number + tags: ["softVersion"] sql: ${TABLE}."VERSION" ;; } diff --git a/metadata-ingestion/tests/integration/lookml/lkml_samples_hive/included_view_file.view.lkml b/metadata-ingestion/tests/integration/lookml/lkml_samples_hive/included_view_file.view.lkml index 6e1ccda22cac7..06439ad14ef88 100644 --- a/metadata-ingestion/tests/integration/lookml/lkml_samples_hive/included_view_file.view.lkml +++ b/metadata-ingestion/tests/integration/lookml/lkml_samples_hive/included_view_file.view.lkml @@ -1,3 +1,3 @@ view: include_able_view { - sql_table_name: looker_schema.include_able ;; + sql_table_name: "looker_schema"."include_able" ;; } diff --git a/metadata-ingestion/tests/integration/lookml/lkml_samples_hive/liquid.view.lkml b/metadata-ingestion/tests/integration/lookml/lkml_samples_hive/liquid.view.lkml index bfeebfba09aa6..4ce62579d08ed 100644 --- a/metadata-ingestion/tests/integration/lookml/lkml_samples_hive/liquid.view.lkml +++ b/metadata-ingestion/tests/integration/lookml/lkml_samples_hive/liquid.view.lkml @@ -1,6 +1,6 @@ view: customer_facts { derived_table: { -sql: + sql: SELECT customer_id, SUM(sale_price) AS lifetime_spend diff --git a/metadata-ingestion/tests/integration/lookml/lkml_samples_hive/nested/fragment_derived.view.lkml b/metadata-ingestion/tests/integration/lookml/lkml_samples_hive/nested/fragment_derived.view.lkml index 284bf4172cfdc..d8397cc355e0e 100644 --- a/metadata-ingestion/tests/integration/lookml/lkml_samples_hive/nested/fragment_derived.view.lkml +++ b/metadata-ingestion/tests/integration/lookml/lkml_samples_hive/nested/fragment_derived.view.lkml @@ -1,9 +1,9 @@ view: fragment_derived_view { derived_table: { - sql: date DATE encode ZSTD, - platform VARCHAR(20) encode ZSTD AS aliased_platform, - country VARCHAR(20) encode ZSTD + sql: SELECT date AS DATE, + platform AS aliased_platform, + country ;; } } diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json index a846e2ca84b09..8813ea532fa2b 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json @@ -170,6 +170,63 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-foo.default-db.my_table,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,project-foo.default-db.my_table,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,project-foo.default-db.my_table,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,project-foo.default-db.my_table,PROD),is_latest)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),is_latest)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,project-foo.default-db.my_table,PROD),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,project-foo.default-db.my_table,PROD),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -428,6 +485,52 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -1299,7 +1402,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "date DATE encode ZSTD, \n platform VARCHAR(20) encode ZSTD AS aliased_platform, \n country VARCHAR(20) encode ZSTD", + "viewLogic": "SELECT date AS DATE,\n platform AS aliased_platform,\n country", "viewLanguage": "sql" } }, @@ -1353,6 +1456,41 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-foo.default-db.fragment_derived_view,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,project-foo.default-db.fragment_derived_view,PROD),date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,project-foo.default-db.fragment_derived_view,PROD),platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),aliased_platform)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,project-foo.default-db.fragment_derived_view,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -1377,7 +1515,7 @@ }, "fields": [ { - "fieldPath": "aliased_platform", + "fieldPath": "date", "nullable": false, "description": "", "label": "", @@ -1388,13 +1526,10 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false }, { - "fieldPath": "country", + "fieldPath": "aliased_platform", "nullable": false, "description": "", "label": "", @@ -1405,13 +1540,10 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false }, { - "fieldPath": "date", + "fieldPath": "country", "nullable": false, "description": "", "label": "", @@ -1422,9 +1554,6 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } ], @@ -1503,7 +1632,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", + "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n order.region='ap-south-1'\n GROUP BY 1", "viewLanguage": "sql" } }, @@ -1557,9 +1686,85 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-foo.default-db.order,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,project-foo.default-db.order,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,project-foo.default-db.order,PROD),sale_price)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),lifetime_spend)" + ], + "confidenceScore": 1.0 + } ] } }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "customer_facts", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lifetime_spend", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json index 47d536fb82409..4bc1a0f2f7da5 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json @@ -170,6 +170,63 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,default-hive-db.my_table,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,default-hive-db.my_table,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,default-hive-db.my_table,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,default-hive-db.my_table,PROD),is_latest)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),is_latest)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,default-hive-db.my_table,PROD),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,default-hive-db.my_table,PROD),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -428,6 +485,52 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -1299,7 +1402,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "date DATE encode ZSTD, \n platform VARCHAR(20) encode ZSTD AS aliased_platform, \n country VARCHAR(20) encode ZSTD", + "viewLogic": "SELECT date AS DATE,\n platform AS aliased_platform,\n country", "viewLanguage": "sql" } }, @@ -1353,6 +1456,41 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,default-hive-db.fragment_derived_view,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,default-hive-db.fragment_derived_view,PROD),date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,default-hive-db.fragment_derived_view,PROD),platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),aliased_platform)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,default-hive-db.fragment_derived_view,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -1377,7 +1515,7 @@ }, "fields": [ { - "fieldPath": "aliased_platform", + "fieldPath": "date", "nullable": false, "description": "", "label": "", @@ -1388,13 +1526,10 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false }, { - "fieldPath": "country", + "fieldPath": "aliased_platform", "nullable": false, "description": "", "label": "", @@ -1405,13 +1540,10 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false }, { - "fieldPath": "date", + "fieldPath": "country", "nullable": false, "description": "", "label": "", @@ -1422,9 +1554,6 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } ], @@ -1503,7 +1632,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", + "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n order.region='ap-south-1'\n GROUP BY 1", "viewLanguage": "sql" } }, @@ -1557,9 +1686,85 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,default-hive-db.order,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,default-hive-db.order,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,default-hive-db.order,PROD),sale_price)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD),lifetime_spend)" + ], + "confidenceScore": 1.0 + } ] } }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "customer_facts", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lifetime_spend", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json index 7e323170e58da..3fd37c4722185 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json @@ -159,6 +159,77 @@ "removed": false } }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),is_latest)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),is_latest)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } + ] + } + }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { "schemaName": "my_view", @@ -403,6 +474,66 @@ "removed": false } }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } + ] + } + }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { "schemaName": "my_derived_view", @@ -1271,7 +1402,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "date DATE encode ZSTD, \n platform VARCHAR(20) encode ZSTD AS aliased_platform, \n country VARCHAR(20) encode ZSTD", + "viewLogic": "SELECT date AS DATE,\n platform AS aliased_platform,\n country", "viewLanguage": "sql" } }, @@ -1314,6 +1445,121 @@ "removed": false } }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD),date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD),platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),aliased_platform)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "fragment_derived_view", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "date", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "aliased_platform", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "country", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { @@ -1386,7 +1632,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", + "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", "viewLanguage": "sql" } }, @@ -1429,20 +1675,6 @@ "removed": false } }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD)", - "type": "VIEW" - } - ] - } - }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json index caefb7b9bcce2..3fd37c4722185 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json @@ -170,6 +170,63 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),is_latest)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),is_latest)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -428,6 +485,52 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -1299,7 +1402,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "date DATE encode ZSTD, \n platform VARCHAR(20) encode ZSTD AS aliased_platform, \n country VARCHAR(20) encode ZSTD", + "viewLogic": "SELECT date AS DATE,\n platform AS aliased_platform,\n country", "viewLanguage": "sql" } }, @@ -1353,6 +1456,41 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD),date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD),platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),aliased_platform)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -1377,7 +1515,7 @@ }, "fields": [ { - "fieldPath": "aliased_platform", + "fieldPath": "date", "nullable": false, "description": "", "label": "", @@ -1388,13 +1526,10 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false }, { - "fieldPath": "country", + "fieldPath": "aliased_platform", "nullable": false, "description": "", "label": "", @@ -1405,13 +1540,10 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false }, { - "fieldPath": "date", + "fieldPath": "country", "nullable": false, "description": "", "label": "", @@ -1422,9 +1554,6 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } ], @@ -1503,7 +1632,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", + "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", "viewLanguage": "sql" } }, @@ -1546,20 +1675,6 @@ "removed": false } }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD)", - "type": "VIEW" - } - ] - } - }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json index c0cec6c261010..bb8a379fdde22 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json @@ -170,6 +170,63 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),is_latest)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),is_latest)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -428,6 +485,52 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -1299,7 +1402,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "date DATE encode ZSTD, \n platform VARCHAR(20) encode ZSTD AS aliased_platform, \n country VARCHAR(20) encode ZSTD", + "viewLogic": "SELECT date AS DATE,\n platform AS aliased_platform,\n country", "viewLanguage": "sql" } }, @@ -1353,6 +1456,41 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.fragment_derived_view,DEV)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.fragment_derived_view,DEV),date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.fragment_derived_view,DEV),platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),aliased_platform)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.fragment_derived_view,DEV),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -1377,7 +1515,7 @@ }, "fields": [ { - "fieldPath": "aliased_platform", + "fieldPath": "date", "nullable": false, "description": "", "label": "", @@ -1388,13 +1526,10 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false }, { - "fieldPath": "country", + "fieldPath": "aliased_platform", "nullable": false, "description": "", "label": "", @@ -1405,13 +1540,10 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false }, { - "fieldPath": "date", + "fieldPath": "country", "nullable": false, "description": "", "label": "", @@ -1422,9 +1554,6 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } ], @@ -1503,7 +1632,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", + "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", "viewLanguage": "sql" } }, @@ -1546,20 +1675,6 @@ "removed": false } }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.order,DEV)", - "type": "VIEW" - } - ] - } - }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json index 31aec97293e5a..b8a2bcc020c34 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json @@ -170,6 +170,63 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),is_latest)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),is_latest)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.my_table,PROD),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -429,6 +486,52 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -1306,7 +1409,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "date DATE encode ZSTD, \n platform VARCHAR(20) encode ZSTD AS aliased_platform, \n country VARCHAR(20) encode ZSTD", + "viewLogic": "SELECT date AS DATE,\n platform AS aliased_platform,\n country", "viewLanguage": "sql" } }, @@ -1360,6 +1463,41 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD),date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD),platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),aliased_platform)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.fragment_derived_view,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -1384,7 +1522,7 @@ }, "fields": [ { - "fieldPath": "aliased_platform", + "fieldPath": "date", "nullable": false, "description": "", "label": "", @@ -1395,13 +1533,10 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false }, { - "fieldPath": "country", + "fieldPath": "aliased_platform", "nullable": false, "description": "", "label": "", @@ -1412,13 +1547,10 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false }, { - "fieldPath": "date", + "fieldPath": "country", "nullable": false, "description": "", "label": "", @@ -1429,9 +1561,6 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } ], @@ -1511,7 +1640,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", + "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", "viewLanguage": "sql" } }, @@ -1554,20 +1683,6 @@ "removed": false } }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,default_db.default_schema.order,PROD)", - "type": "VIEW" - } - ] - } - }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json b/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json index 5c43d1dc5ceaf..8c3504a736490 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json @@ -142,30 +142,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -194,6 +170,63 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),is_latest)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),is_latest)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -350,13 +383,19 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "View" + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } ] } }, @@ -370,12 +409,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "subTypes", "aspect": { "json": { - "materialized": false, - "viewLogic": "view: owners {\n dimension: id {\n primary_key: yes\n sql: ${TABLE}.id ;;\n }\n dimension: owner_name {\n sql: ${TABLE}.owner_name ;;\n }\n}", - "viewLanguage": "lookml" + "typeNames": [ + "View" + ] } }, "systemMetadata": { @@ -388,10 +427,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "viewProperties", "aspect": { "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + "materialized": false, + "viewLogic": "view: owners {\n dimension: id {\n primary_key: yes\n sql: ${TABLE}.id ;;\n }\n dimension: owner_name {\n sql: ${TABLE}.owner_name ;;\n }\n}", + "viewLanguage": "lookml" } }, "systemMetadata": { @@ -404,18 +445,10 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] + "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" } }, "systemMetadata": { @@ -568,13 +601,19 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "View" + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } ] } }, @@ -588,12 +627,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "subTypes", "aspect": { "json": { - "materialized": false, - "viewLogic": "SELECT\n is_latest,\n country,\n city,\n timestamp,\n measurement\n FROM\n my_table", - "viewLanguage": "sql" + "typeNames": [ + "View" + ] } }, "systemMetadata": { @@ -606,10 +645,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "viewProperties", "aspect": { "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + "materialized": false, + "viewLogic": "SELECT\n is_latest,\n country,\n city,\n timestamp,\n measurement\n FROM\n my_table", + "viewLanguage": "sql" } }, "systemMetadata": { @@ -622,18 +663,10 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] + "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" } }, "systemMetadata": { @@ -670,6 +703,63 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV),is_latest)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD),is_latest)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -824,6 +914,30 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_same_name_views_different_file_path.json b/metadata-ingestion/tests/integration/lookml/lookml_same_name_views_different_file_path.json index a5357ccada8a3..b86f7703e6f5e 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_same_name_views_different_file_path.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_same_name_views_different_file_path.json @@ -142,36 +142,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path1.foo.view.my_view,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - }, - { - "id": "path1" - }, - { - "id": "foo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -200,6 +170,63 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path1.foo.view.my_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path1.foo.view.my_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),is_latest)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path1.foo.view.my_view,PROD),is_latest)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path1.foo.view.my_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path1.foo.view.my_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -356,13 +383,25 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path2.foo.view.my_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path1.foo.view.my_view,PROD)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "View" + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + }, + { + "id": "path1" + }, + { + "id": "foo" + } ] } }, @@ -376,12 +415,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path2.foo.view.my_view,PROD)", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "subTypes", "aspect": { "json": { - "materialized": false, - "viewLogic": "SELECT\n is_latest,\n country,\n city,\n timestamp,\n measurement\n FROM\n my_table", - "viewLanguage": "sql" + "typeNames": [ + "View" + ] } }, "systemMetadata": { @@ -394,10 +433,12 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path2.foo.view.my_view,PROD)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "viewProperties", "aspect": { "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + "materialized": false, + "viewLogic": "SELECT\n is_latest,\n country,\n city,\n timestamp,\n measurement\n FROM\n my_table", + "viewLanguage": "sql" } }, "systemMetadata": { @@ -410,24 +451,10 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path2.foo.view.my_view,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - }, - { - "id": "path2" - }, - { - "id": "foo" - } - ] + "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" } }, "systemMetadata": { @@ -464,6 +491,52 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path2.foo.view.my_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),is_latest)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path2.foo.view.my_view,PROD),is_latest)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path2.foo.view.my_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path2.foo.view.my_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -597,6 +670,36 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.path2.foo.view.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + }, + { + "id": "path2" + }, + { + "id": "foo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { diff --git a/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json b/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json index 16e901125e451..7265ee3c6c62b 100644 --- a/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json +++ b/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json @@ -167,9 +167,66 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,..my_table,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,my_table,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,my_table,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.foo.view.my_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,my_table,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.foo.view.my_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,my_table,PROD),is_latest)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.foo.view.my_view,PROD),is_latest)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,my_table,PROD),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.foo.view.my_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,my_table,PROD),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.foo.view.my_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -428,9 +485,55 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.bar.view.my_view,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.foo.view.my_view,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.foo.view.my_view,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.bar.view.my_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.foo.view.my_view,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.bar.view.my_derived_view,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.foo.view.my_view,PROD),timestamp)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.bar.view.my_derived_view,PROD),timestamp)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.foo.view.my_view,PROD),measurement)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.bar.view.my_derived_view,PROD),average_measurement)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -668,7 +771,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,.looker_schema.include_able,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.looker_schema.include_able,PROD)", "type": "VIEW" } ] @@ -797,7 +900,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,.looker_schema.events,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.looker_schema.events,PROD)", "type": "VIEW" } ] @@ -926,7 +1029,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,.looker_schema.events,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.looker_schema.events,PROD)", "type": "VIEW" } ], @@ -934,7 +1037,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.looker_schema.events,PROD),additional_measure)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.looker_schema.events,PROD),additional_measure)" ], "downstreamType": "FIELD", "downstreams": [ @@ -1113,7 +1216,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,..autodetect_sql_name_based_on_view_name,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,..autodetect_sql_name_based_on_view_name,PROD)", "type": "VIEW" } ] @@ -1242,7 +1345,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,.looker_schema.include_able,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.looker_schema.include_able,PROD)", "type": "VIEW" } ] @@ -1320,7 +1423,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "date DATE encode ZSTD, \n platform VARCHAR(20) encode ZSTD AS aliased_platform, \n country VARCHAR(20) encode ZSTD", + "viewLogic": "SELECT date AS DATE,\n platform AS aliased_platform,\n country", "viewLanguage": "sql" } }, @@ -1371,9 +1474,44 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,..fragment_derived_view,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,fragment_derived_view,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,fragment_derived_view,PROD),date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.nested.fragment_derived.view.fragment_derived_view,PROD),date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,fragment_derived_view,PROD),platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.nested.fragment_derived.view.fragment_derived_view,PROD),aliased_platform)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,fragment_derived_view,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.nested.fragment_derived.view.fragment_derived_view,PROD),country)" + ], + "confidenceScore": 1.0 + } ] } }, @@ -1398,7 +1536,7 @@ }, "fields": [ { - "fieldPath": "aliased_platform", + "fieldPath": "date", "nullable": false, "description": "", "label": "", @@ -1409,13 +1547,10 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false }, { - "fieldPath": "country", + "fieldPath": "aliased_platform", "nullable": false, "description": "", "label": "", @@ -1426,13 +1561,10 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false }, { - "fieldPath": "date", + "fieldPath": "country", "nullable": false, "description": "", "label": "", @@ -1443,9 +1575,6 @@ }, "nativeDataType": "unknown", "recursive": false, - "globalTags": { - "tags": [] - }, "isPartOfKey": false } ], @@ -1527,7 +1656,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n {% condition order_region %} order.region {% endcondition %}\n GROUP BY 1", + "viewLogic": "SELECT\n customer_id,\n SUM(sale_price) AS lifetime_spend\n FROM\n order\n WHERE\n order.region='ap-south-1'\n GROUP BY 1", "viewLanguage": "sql" } }, @@ -1578,12 +1707,88 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,..order,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,order,PROD)", "type": "VIEW" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,order,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.liquid.view.customer_facts,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,order,PROD),sale_price)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.liquid.view.customer_facts,PROD),lifetime_spend)" + ], + "confidenceScore": 1.0 + } ] } }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "customer_facts", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lifetime_spend", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "unknown", + "recursive": false, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { @@ -1707,7 +1912,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,.ecommerce.ability,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.ecommerce.ability,PROD)", "type": "VIEW" } ], @@ -1715,7 +1920,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.ecommerce.ability,PROD),pk)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.ecommerce.ability,PROD),pk)" ], "downstreamType": "FIELD", "downstreams": [ @@ -1726,7 +1931,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.ecommerce.ability,PROD),count)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.ecommerce.ability,PROD),count)" ], "downstreamType": "FIELD", "downstreams": [ @@ -1926,7 +2131,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,..owners,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,..owners,PROD)", "type": "VIEW" } ], @@ -1934,7 +2139,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,..owners,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,..owners,PROD),id)" ], "downstreamType": "FIELD", "downstreams": [ @@ -1945,7 +2150,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,..owners,PROD),owner_name)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,..owners,PROD),owner_name)" ], "downstreamType": "FIELD", "downstreams": [ @@ -1956,7 +2161,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,..owners,PROD),owner_name)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,..owners,PROD),owner_name)" ], "downstreamType": "FIELD", "downstreams": [ @@ -2462,7 +2667,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:conn,.flightstats.accidents,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.flightstats.accidents,PROD)", "type": "VIEW" } ], @@ -2470,7 +2675,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:conn,.flightstats.accidents,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.flightstats.accidents,PROD),id)" ], "downstreamType": "FIELD", "downstreams": [ diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py index 1c1f0fec3eebb..9e051995d0b94 100644 --- a/metadata-ingestion/tests/integration/lookml/test_lookml.py +++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py @@ -12,6 +12,10 @@ from datahub.configuration.common import PipelineExecutionError from datahub.ingestion.run.pipeline import Pipeline from datahub.ingestion.source.file import read_metadata_file +from datahub.ingestion.source.looker.looker_template_language import ( + SpecialVariable, + resolve_liquid_variable, +) from datahub.ingestion.source.looker.lookml_source import ( LookerModel, LookerRefinementResolver, @@ -40,12 +44,13 @@ def get_default_recipe(output_file_path, base_folder_path): "type": "lookml", "config": { "base_folder": base_folder_path, - "connection_to_platform_map": {"my_connection": "conn"}, + "connection_to_platform_map": {"my_connection": "postgres"}, "parse_table_names_from_sql": True, "tag_measures_and_dimensions": False, "project_name": "lkml_samples", "model_pattern": {"deny": ["data2"]}, "emit_reachable_views_only": False, + "liquid_variable": {"order_region": "ap-south-1"}, }, }, "sink": { @@ -63,9 +68,9 @@ def test_lookml_ingest(pytestconfig, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml" mce_out_file = "expected_output.json" - # Note this config below is known to create "bad" lineage since the config author has not provided enough information - # to resolve relative table names (which are not fully qualified) - # We keep this check just to validate that ingestion doesn't croak on this config + # Note this config below is known to create "bad" lineage since the config author has not provided enough + # information to resolve relative table names (which are not fully qualified) We keep this check just to validate + # that ingestion doesn't croak on this config pipeline = Pipeline.create( get_default_recipe( @@ -488,6 +493,9 @@ def ingestion_test( "model_pattern": {"deny": ["data2"]}, "emit_reachable_views_only": False, "process_refinements": False, + "liquid_variable": { + "order_region": "ap-south-1", + }, }, }, "sink": { @@ -868,6 +876,7 @@ def test_manifest_parser(pytestconfig: pytest.Config) -> None: @freeze_time(FROZEN_TIME) def test_duplicate_field_ingest(pytestconfig, tmp_path, mock_time): + test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml" mce_out_file = "duplicate_ingest_mces_output.json" @@ -887,3 +896,139 @@ def test_duplicate_field_ingest(pytestconfig, tmp_path, mock_time): output_path=tmp_path / mce_out_file, golden_path=golden_path, ) + + +@freeze_time(FROZEN_TIME) +def test_view_to_view_lineage_and_liquid_template(pytestconfig, tmp_path, mock_time): + test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml" + mce_out_file = "vv_lineage_liquid_template_golden.json" + + new_recipe = get_default_recipe( + f"{tmp_path}/{mce_out_file}", + f"{test_resources_dir}/vv-lineage-and-liquid-templates", + ) + + new_recipe["source"]["config"]["liquid_variable"] = { + "_user_attributes": { + "looker_env": "dev", + "dev_database_prefix": "employee", + "dev_schema_prefix": "public", + }, + "dw_eff_dt_date": { + "_is_selected": True, + }, + "source_region": "ap-south-1", + } + + pipeline = Pipeline.create(new_recipe) + pipeline.run() + pipeline.pretty_print_summary() + pipeline.raise_from_status(raise_warnings=True) + + golden_path = test_resources_dir / "vv_lineage_liquid_template_golden.json" + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / mce_out_file, + golden_path=golden_path, + ) + + +@freeze_time(FROZEN_TIME) +def test_special_liquid_variables(): + text: str = """ + SELECT + employee_id, + employee_name, + {% if dw_eff_dt_date._is_selected or finance_dw_eff_dt_date._is_selected %} + prod_core.data.r_metric_summary_v2 + {% elsif dw_eff_dt_week._is_selected or finance_dw_eff_dt_week._in_query %} + prod_core.data.r_metric_summary_v3 + {% elsif dw_eff_dt_week._is_selected or finance_dw_eff_dt_week._is_filtered %} + prod_core.data.r_metric_summary_v4 + {% else %} + 'default_table' as source + {% endif %}, + employee_income + FROM source_table + """ + input_liquid_variable: dict = {} + + expected_liquid_variable: dict = { + **input_liquid_variable, + "dw_eff_dt_date": {"_is_selected": True}, + "finance_dw_eff_dt_date": {"_is_selected": True}, + "dw_eff_dt_week": {"_is_selected": True}, + "finance_dw_eff_dt_week": { + "_in_query": True, + "_is_filtered": True, + }, + } + + actual_liquid_variable = SpecialVariable( + input_liquid_variable + ).liquid_variable_with_default(text) + assert ( + expected_liquid_variable == actual_liquid_variable + ) # Here new keys with default value should get added + + # change input + input_liquid_variable = { + "finance_dw_eff_dt_week": {"_is_filtered": False}, + } + + expected_liquid_variable = { + **input_liquid_variable, + "dw_eff_dt_date": {"_is_selected": True}, + "finance_dw_eff_dt_date": {"_is_selected": True}, + "dw_eff_dt_week": {"_is_selected": True}, + "finance_dw_eff_dt_week": { + "_in_query": True, + "_is_filtered": False, + }, + } + + actual_liquid_variable = SpecialVariable( + input_liquid_variable + ).liquid_variable_with_default(text) + assert ( + expected_liquid_variable == actual_liquid_variable + ) # should not overwrite the actual value present in + # input_liquid_variable + + # Match template after resolution of liquid variables + actual_text = resolve_liquid_variable( + text=text, + liquid_variable=input_liquid_variable, + ) + + expected_text: str = ( + "\n SELECT\n employee_id,\n employee_name,\n \n " + "prod_core.data.r_metric_summary_v2\n ,\n employee_income\n FROM " + "source_table\n " + ) + assert actual_text == expected_text + + +@freeze_time(FROZEN_TIME) +def test_field_tag_ingest(pytestconfig, tmp_path, mock_time): + test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml" + mce_out_file = "field_tag_mces_output.json" + + new_recipe = get_default_recipe( + f"{tmp_path}/{mce_out_file}", + f"{test_resources_dir}/lkml_samples_duplicate_field", + ) + + new_recipe["source"]["config"]["tag_measures_and_dimensions"] = True + + pipeline = Pipeline.create(new_recipe) + pipeline.run() + pipeline.pretty_print_summary() + pipeline.raise_from_status(raise_warnings=True) + + golden_path = test_resources_dir / "field_tag_ingestion_golden.json" + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / mce_out_file, + golden_path=golden_path, + ) diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/activity_logs.view.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/activity_logs.view.lkml new file mode 100644 index 0000000000000..f0e2dec6e4678 --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/activity_logs.view.lkml @@ -0,0 +1,18 @@ +view: activity_logs { + sql_table_name: + {% if _user_attributes['looker_env'] == 'dev' %} + {{ _user_attributes['dev_database_prefix'] }}analytics.{{ _user_attributes['dev_schema_prefix'] }}staging_app.stg_app__activity_logs + {% elsif _user_attributes['looker_env'] == 'prod' %} + analytics.staging_app.stg_app__activity_logs + {% else %} + analytics.staging_app.stg_app__activity_logs + {% endif %} + ;; + + dimension: generated_message_id { + group_label: "IDs" + primary_key: yes + type: number + sql: ${TABLE}."GENERATED_MESSAGE_ID" ;; + } +} diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml new file mode 100644 index 0000000000000..ea55512c5ca06 --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml @@ -0,0 +1,22 @@ +connection: "my_connection" + +include: "activity_logs.view.lkml" +include: "employee_income_source.view.lkml" +include: "employee_total_income.view.lkml" +include: "top_10_employee_income_source.view.lkml" +include: "employee_tax_report.view.lkml" + +explore: activity_logs { +} + +explore: employee_income_source { +} + +explore: employee_total_income { +} + +explore: top_10_employee_income_source { +} + +explore: employee_tax_report { +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/employee_income_source.view.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/employee_income_source.view.lkml new file mode 100644 index 0000000000000..f4a443ab11537 --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/employee_income_source.view.lkml @@ -0,0 +1,40 @@ +view: employee_income_source { + derived_table: { + sql: SELECT + employee_id, + employee_name, + {% if dw_eff_dt_date._is_selected or finance_dw_eff_dt_date._is_selected %} + prod_core.data.r_metric_summary_v2 + {% elsif dw_eff_dt_week._is_selected or finance_dw_eff_dt_week._is_selected %} + prod_core.data.r_metric_summary_v3 + {% else %} + 'default_table' as source + {% endif %}, + employee_income + FROM source_table + WHERE + {% condition source_region %} source_table.region {% endcondition %} + ;; + } + + dimension: id { + type: number + sql: ${TABLE}.employee_id;; + } + + dimension: name { + type: string + sql: ${TABLE}.employee_name;; + } + + dimension: source { + type: string + sql: ${TABLE}.source ;; + } + + dimension: income { + type: number + sql: ${TABLE}.employee_income ;; + } + +} diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/employee_tax_report.view.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/employee_tax_report.view.lkml new file mode 100644 index 0000000000000..6608921e6f095 --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/employee_tax_report.view.lkml @@ -0,0 +1,18 @@ +view: employee_tax_report { + sql_table_name: data-warehouse.finance.form-16;; + + dimension: id { + type: number + sql: ${TABLE}.id;; + } + + dimension: name { + type: string + sql: ${TABLE}.name;; + } + + measure: taxable_income { + type: sum + sql: ${TABLE}.tax;; + } +} diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/employee_total_income.view.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/employee_total_income.view.lkml new file mode 100644 index 0000000000000..18a1ab660b3a1 --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/employee_total_income.view.lkml @@ -0,0 +1,18 @@ +view: employee_total_income { + sql_table_name: ${employee_income_source.SQL_TABLE_NAME} ;; + + dimension: id { + type: number + sql: ${TABLE}.id;; + } + + dimension: name { + type: string + sql: ${TABLE}.name;; + } + + measure: total_income { + type: sum + sql: ${TABLE}.income;; + } +} diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/top_10_employee_income_source.view.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/top_10_employee_income_source.view.lkml new file mode 100644 index 0000000000000..5371ed5e3ca56 --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/top_10_employee_income_source.view.lkml @@ -0,0 +1,26 @@ +view: top_10_employee_income_source { + derived_table: { + sql: SELECT id, + name, + source + FROM ${employee_income_source.SQL_TABLE_NAME} + ORDER BY source desc + LIMIT 10 + ;; + } + + dimension: id { + type: number + sql: ${TABLE}.id ;; + } + + dimension: name { + type: string + sql: ${TABLE}.name ;; + } + + dimension: source { + type: string + sql: ${TABLE}.source ;; + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json new file mode 100644 index 0000000000000..75cd50c5c6059 --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json @@ -0,0 +1,1335 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "looker", + "env": "PROD", + "project_name": "lkml_samples" + }, + "name": "lkml_samples" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:looker" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "LookML Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.activity_logs,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.activity_logs,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "view: activity_logs {\n sql_table_name:\n {% if _user_attributes['looker_env'] == 'dev' %}\n {{ _user_attributes['dev_database_prefix'] }}analytics.{{ _user_attributes['dev_schema_prefix'] }}staging_app.stg_app__activity_logs\n {% elsif _user_attributes['looker_env'] == 'prod' %}\n analytics.staging_app.stg_app__activity_logs\n {% else %}\n analytics.staging_app.stg_app__activity_logs\n {% endif %}\n ;;\n\n dimension: generated_message_id {\n group_label: \"IDs\"\n primary_key: yes\n type: number\n sql: ${TABLE}.\"GENERATED_MESSAGE_ID\" ;;\n }\n}\n", + "viewLanguage": "lookml" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.activity_logs,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.activity_logs,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Develop/lkml_samples/" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,employeeanalytics.publicstaging_app.stg_app__activity_logs,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,employeeanalytics.publicstaging_app.stg_app__activity_logs,PROD),generated_message_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.activity_logs,PROD),generated_message_id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "activity_logs", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "generated_message_id", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": true + } + ], + "primaryKeys": [ + "generated_message_id" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "looker.file.path": "activity_logs.view.lkml", + "looker.model": "data" + }, + "name": "activity_logs", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.activity_logs,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "SELECT\n employee_id,\n employee_name,\n \n prod_core.data.r_metric_summary_v2\n ,\n employee_income\n FROM source_table\n WHERE\n source_table.region='ap-south-1'", + "viewLanguage": "sql" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Develop/lkml_samples/" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,source_table,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,source_table,PROD),employee_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,source_table,PROD),employee_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD),name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,source_table,PROD),source)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD),source)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,source_table,PROD),employee_income)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD),income)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "employee_income_source", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "source", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "income", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "looker.file.path": "employee_income_source.view.lkml", + "looker.model": "data" + }, + "name": "employee_income_source", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_total_income,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_total_income,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "view: employee_total_income {\n sql_table_name: ${employee_income_source.SQL_TABLE_NAME} ;;\n\n dimension: id {\n type: number\n sql: ${TABLE}.id;;\n }\n\n dimension: name {\n type: string\n sql: ${TABLE}.name;;\n }\n\n measure: total_income {\n type: sum\n sql: ${TABLE}.income;;\n }\n}\n", + "viewLanguage": "lookml" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_total_income,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_total_income,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Develop/lkml_samples/" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_total_income,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_total_income,PROD),name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD),income)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_total_income,PROD),total_income)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "employee_total_income", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "total_income", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "sum", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Measure" + } + ] + }, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "looker.file.path": "employee_total_income.view.lkml", + "looker.model": "data" + }, + "name": "employee_total_income", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_total_income,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "SELECT id,\n name,\n source\n FROM ${employee_income_source.SQL_TABLE_NAME}\n ORDER BY source desc\n LIMIT 10", + "viewLanguage": "sql" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Develop/lkml_samples/" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD),name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_income_source,PROD),source)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD),source)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "top_10_employee_income_source", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "source", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "looker.file.path": "top_10_employee_income_source.view.lkml", + "looker.model": "data" + }, + "name": "top_10_employee_income_source", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.top_10_employee_income_source,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_tax_report,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_tax_report,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "view: employee_tax_report {\n sql_table_name: data-warehouse.finance.form-16;;\n\n dimension: id {\n type: number\n sql: ${TABLE}.id;;\n }\n\n dimension: name {\n type: string\n sql: ${TABLE}.name;;\n }\n\n measure: taxable_income {\n type: sum\n sql: ${TABLE}.tax;;\n }\n}\n", + "viewLanguage": "lookml" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_tax_report,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_tax_report,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Develop/lkml_samples/" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,data-warehouse.finance.form-16,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,data-warehouse.finance.form-16,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_tax_report,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,data-warehouse.finance.form-16,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_tax_report,PROD),name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,data-warehouse.finance.form-16,PROD),tax)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_tax_report,PROD),taxable_income)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "employee_tax_report", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "taxable_income", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "sum", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Measure" + } + ] + }, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "looker.file.path": "employee_tax_report.view.lkml", + "looker.model": "data" + }, + "name": "employee_tax_report", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.employee_tax_report,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Dimension", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Dimension" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Measure", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Measure" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json b/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json index 9bab81eaa228b..976fe5f77c929 100644 --- a/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json +++ b/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json @@ -15,8 +15,8 @@ "description": "", "lastModified": { "created": { - "time": 1639417592792, - "actor": "urn:li:corpuser:admin@metabase.com" + "time": 0, + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1639417592792, @@ -49,6 +49,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -61,7 +62,8 @@ "systemMetadata": { "lastObserved": 1636614000000, "runId": "metabase-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } }, { @@ -80,8 +82,8 @@ "description": "", "lastModified": { "created": { - "time": 1636614000000, - "actor": "urn:li:corpuser:admin@metabase.com" + "time": 0, + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1636614000000, @@ -105,6 +107,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -117,7 +120,8 @@ "systemMetadata": { "lastObserved": 1636614000000, "runId": "metabase-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } }, { @@ -136,8 +140,8 @@ "description": "", "lastModified": { "created": { - "time": 1685628119636, - "actor": "urn:li:corpuser:john.doe@example.com" + "time": 0, + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1685628119636, @@ -164,6 +168,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -176,7 +181,8 @@ "systemMetadata": { "lastObserved": 1636614000000, "runId": "metabase-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } }, { @@ -216,6 +222,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -228,7 +235,8 @@ "systemMetadata": { "lastObserved": 1636614000000, "runId": "metabase-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } }, { @@ -268,6 +276,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -280,7 +289,8 @@ "systemMetadata": { "lastObserved": 1636614000000, "runId": "metabase-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } }, { @@ -320,6 +330,61 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1636614000000, + "runId": "metabase-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(metabase,20)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Dashboard 2", + "description": "", + "charts": [ + "urn:li:chart:(metabase,1)", + "urn:li:chart:(metabase,2)", + "urn:li:chart:(metabase,3)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1705398694904, + "actor": "urn:li:corpuser:admin@metabase.com" + }, + "lastModified": { + "time": 1705398694904, + "actor": "urn:li:corpuser:admin@metabase.com" + } + }, + "dashboardUrl": "http://localhost:3000/dashboard/20" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:admin@metabase.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -332,7 +397,8 @@ "systemMetadata": { "lastObserved": 1636614000000, "runId": "metabase-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } }, { @@ -348,7 +414,8 @@ "systemMetadata": { "lastObserved": 1636614000000, "runId": "metabase-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } }, { @@ -364,7 +431,8 @@ "systemMetadata": { "lastObserved": 1636614000000, "runId": "metabase-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } }, { @@ -380,7 +448,8 @@ "systemMetadata": { "lastObserved": 1636614000000, "runId": "metabase-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } }, { @@ -396,7 +465,8 @@ "systemMetadata": { "lastObserved": 1636614000000, "runId": "metabase-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } }, { @@ -412,7 +482,8 @@ "systemMetadata": { "lastObserved": 1636614000000, "runId": "metabase-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/test_metabase.py b/metadata-ingestion/tests/integration/metabase/test_metabase.py index 5c433f14f380f..2d67f0ca5223f 100644 --- a/metadata-ingestion/tests/integration/metabase/test_metabase.py +++ b/metadata-ingestion/tests/integration/metabase/test_metabase.py @@ -163,7 +163,7 @@ def test_pipeline(pytestconfig, tmp_path): @freeze_time(FROZEN_TIME) -def test_mode_ingest_success( +def test_metabase_ingest_success( pytestconfig, tmp_path, test_pipeline, mock_datahub_graph, default_json_response_map ): with patch( @@ -260,7 +260,7 @@ def test_stateful_ingestion( @freeze_time(FROZEN_TIME) -def test_mode_ingest_failure(pytestconfig, tmp_path, default_json_response_map): +def test_metabase_ingest_failure(pytestconfig, tmp_path, default_json_response_map): with patch( "datahub.ingestion.source.metabase.requests.session", side_effect=MockResponse.build_mocked_requests_failure( diff --git a/metadata-ingestion/tests/integration/mode/test_mode.py b/metadata-ingestion/tests/integration/mode/test_mode.py index def7277494fe7..0346767b05d25 100644 --- a/metadata-ingestion/tests/integration/mode/test_mode.py +++ b/metadata-ingestion/tests/integration/mode/test_mode.py @@ -139,7 +139,3 @@ def test_mode_ingest_failure(pytestconfig, tmp_path): except PipelineExecutionError as exec_error: assert exec_error.args[0] == "Source reported errors" assert len(exec_error.args[1].failures) == 1 - assert ( - list(exec_error.args[1].failures.keys())[0] - == "mode-report-75737b70402e" - ) diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json index 4ca5cc4afd00d..29e9ccebf067e 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json @@ -390,7 +390,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, @@ -1709,7 +1709,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json index 4a08c6658c395..937cad0b9ec17 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json @@ -483,7 +483,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json index b65ba71c80159..501ec284097b3 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json @@ -780,7 +780,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, @@ -2213,7 +2213,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json index 43a5aad3cd967..74779ac7a0577 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json @@ -382,7 +382,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json index cf448776ba414..442dfd5c8c082 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json @@ -482,7 +482,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json index 566e405a03abb..2c4ff6ee851f4 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json @@ -382,7 +382,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json index 4f530acb11c80..0aea8514559ec 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json @@ -465,7 +465,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json index 37ba034b08fe2..22a00236af8c6 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json @@ -382,7 +382,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json index 12ce114482eee..cf5d4df460e23 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json @@ -382,7 +382,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_profiling.json b/metadata-ingestion/tests/integration/powerbi/golden_test_profiling.json new file mode 100644 index 0000000000000..580a8d1a1db11 --- /dev/null +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_profiling.json @@ -0,0 +1,170 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1645599600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "articles", + "description": "Library Dataset", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1645599600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1645599600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 542300, + "columnCount": 4, + "fieldProfiles": [ + { + "fieldPath": "link", + "uniqueCount": 15, + "min": "3", + "max": "34333", + "sampleValues": [ + "http://example.org", + "http://example.org/111/22/foo", + "http://example.org/111/22" + ] + }, + { + "fieldPath": "description", + "uniqueCount": 15, + "min": "3", + "max": "34333", + "sampleValues": [ + "this is a sample", + "this describes content", + "sample, this is" + ] + }, + { + "fieldPath": "topic", + "uniqueCount": 15, + "min": "3", + "max": "34333", + "sampleValues": [ + "urgent matters", + "urgent matters", + "normal matters" + ] + }, + { + "fieldPath": "view_count", + "uniqueCount": 15, + "min": "3", + "max": "34333", + "sampleValues": [ + "123455", + "123455", + "123455" + ] + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1645599600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1645599600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1645599600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.articles,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,mics.public.order_date,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1645599600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json index 43092c4e20acb..cce9e3f8755ba 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json @@ -382,7 +382,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, @@ -1493,7 +1493,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json index 3c4499765e525..5e244e0e0f93f 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json @@ -382,7 +382,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json index 0c6239ae8d331..8fd8989b81122 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json @@ -490,7 +490,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py b/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py index bb572fe3a7ae5..91c6082524389 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py +++ b/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py @@ -283,7 +283,7 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None "name": "job-history", "source": [ { - "expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.GSLAB.COM", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1', + "expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.domain.com", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1', } ], "datasourceUsages": [ diff --git a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py index 0aa290cecd8fd..e98b64c011fd8 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py +++ b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py @@ -35,7 +35,7 @@ 'let\n Source = Sql.Database("AUPRDWHDB", "COMMOPSDB", [Query="Select#(lf)*,#(lf)concat((UPPER(REPLACE(SALES_SPECIALIST,\'-\',\'\'))),#(lf)LEFT(CAST(INVOICE_DATE AS DATE),4)+LEFT(RIGHT(CAST(INVOICE_DATE AS DATE),5),2)) AS AGENT_KEY,#(lf)CASE#(lf) WHEN CLASS = \'Software\' and (NOT(PRODUCT in (\'ADV\', \'Adv\') and left(ACCOUNT_ID,2)=\'10\') #(lf) or V_ENTERPRISE_INVOICED_REVENUE.TYPE = \'Manual Adjustment\') THEN INVOICE_AMOUNT#(lf) WHEN V_ENTERPRISE_INVOICED_REVENUE.TYPE IN (\'Recurring\',\'0\') THEN INVOICE_AMOUNT#(lf) ELSE 0#(lf)END as SOFTWARE_INV#(lf)#(lf)from V_ENTERPRISE_INVOICED_REVENUE", CommandTimeout=#duration(0, 1, 30, 0)]),\n #"Added Conditional Column" = Table.AddColumn(Source, "Services", each if [CLASS] = "Services" then [INVOICE_AMOUNT] else 0),\n #"Added Custom" = Table.AddColumn(#"Added Conditional Column", "Advanced New Sites", each if [PRODUCT] = "ADV"\nor [PRODUCT] = "Adv"\nthen [NEW_SITE]\nelse 0)\nin\n #"Added Custom"', 'let\n Source = Snowflake.Databases("ghh48144.snowflakefakecomputing.com","GSL_TEST_WH",[Role="ACCOUNTADMIN"]),\n Source2 = PostgreSQL.Database("localhost", "mics"),\n public_order_date = Source2{[Schema="public",Item="order_date"]}[Data],\n GSL_TEST_DB_Database = Source{[Name="GSL_TEST_DB",Kind="Database"]}[Data],\n PUBLIC_Schema = GSL_TEST_DB_Database{[Name="PUBLIC",Kind="Schema"]}[Data],\n SALES_ANALYST_VIEW_View = PUBLIC_Schema{[Name="SALES_ANALYST_VIEW",Kind="View"]}[Data],\n two_source_table = Table.Combine({public_order_date, SALES_ANALYST_VIEW_View})\n in\n two_source_table', 'let\n Source = PostgreSQL.Database("localhost" , "mics" ),\n public_order_date = Source{[Schema="public",Item="order_date"]}[Data] \n in \n public_order_date', - 'let\n Source = Oracle.Database("localhost:1521/salesdb.GSLAB.COM", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1', + 'let\n Source = Oracle.Database("localhost:1521/salesdb.domain.com", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1', 'let\n Source = Sql.Database("localhost", "library"),\n dbo_book_issue = Source{[Schema="dbo",Item="book_issue"]}[Data]\n in dbo_book_issue', 'let\n Source = Snowflake.Databases("ghh48144.snowflakefakecomputing.com","GSL_TEST_WH",[Role="ACCOUNTADMIN"]),\n GSL_TEST_DB_Database = Source{[Name="GSL_TEST_DB",Kind="Database"]}[Data],\n PUBLIC_Schema = GSL_TEST_DB_Database{[Name="PUBLIC",Kind="Schema"]}[Data],\n SALES_FORECAST_Table = PUBLIC_Schema{[Name="SALES_FORECAST",Kind="Table"]}[Data],\n SALES_ANALYST_Table = PUBLIC_Schema{[Name="SALES_ANALYST",Kind="Table"]}[Data],\n RESULT = Table.Combine({SALES_FORECAST_Table, SALES_ANALYST_Table})\n\nin\n RESULT', 'let\n Source = GoogleBigQuery.Database(),\n #"seraphic-music-344307" = Source{[Name="seraphic-music-344307"]}[Data],\n school_dataset_Schema = #"seraphic-music-344307"{[Name="school_dataset",Kind="Schema"]}[Data],\n first_Table = school_dataset_Schema{[Name="first",Kind="Table"]}[Data]\nin\n first_Table', diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py index fafa02d5301e0..06cc40fe7b24c 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py +++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py @@ -393,7 +393,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "name": "job-history", "source": [ { - "expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.GSLAB.COM", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1', + "expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.domain.com", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1', } ], "datasourceUsages": [ diff --git a/metadata-ingestion/tests/integration/powerbi/test_profiling.py b/metadata-ingestion/tests/integration/powerbi/test_profiling.py new file mode 100644 index 0000000000000..7955386de8940 --- /dev/null +++ b/metadata-ingestion/tests/integration/powerbi/test_profiling.py @@ -0,0 +1,338 @@ +import logging +import sys +from typing import Any, Dict +from unittest import mock + +from freezegun import freeze_time + +from datahub.ingestion.run.pipeline import Pipeline +from tests.test_helpers import mce_helpers + +FROZEN_TIME = "2022-02-23 07:00:00" + + +def scan_init_response(request, context): + # Request mock is passing POST input in the form of workspaces= + workspace_id = request.text.split("=")[1] + + w_id_vs_response: Dict[str, Any] = { + "64ED5CAD-7C10-4684-8180-826122881108": { + "id": "4674efd1-603c-4129-8d82-03cf2be05aff" + } + } + + return w_id_vs_response[workspace_id] + + +def admin_datasets_response(request, context): + return { + "value": [ + { + "id": "05169CD2-E713-41E6-9600-1D8066D95445", + "name": "library-dataset", + "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445", + } + ] + } + + +def execute_queries_response(request, context): + query = request.json()["queries"][0]["query"] + if "unique_count" in query: + return { + "results": [ + { + "tables": [ + { + "rows": [ + { + "[min]": 3, + "[max]": 34333, + "[unique_count]": 15, + }, + ] + } + ] + } + ], + } + elif "COUNTROWS" in query: + return { + "results": [ + { + "tables": [ + { + "rows": [ + { + "[count]": 542300, + }, + ] + } + ] + } + ], + } + elif "TOPN" in query: + return { + "results": [ + { + "tables": [ + { + "rows": [ + { + "[link]": "http://example.org", + "[description]": "this is a sample", + "[topic]": "urgent matters", + "[view_count]": 123455, + }, + { + "[link]": "http://example.org/111/22/foo", + "[description]": "this describes content", + "[topic]": "urgent matters", + "[view_count]": 123455, + }, + { + "[link]": "http://example.org/111/22", + "[description]": "sample, this is", + "[topic]": "normal matters", + "[view_count]": 123455, + }, + ] + } + ] + } + ], + } + + +def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None: + api_vs_response = { + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets": { + "method": "GET", + "status_code": 200, + "json": admin_datasets_response, + }, + "https://api.powerbi.com/v1.0/myorg/groups?%24top=1000&%24skip=0&%24filter=type+eq+%27Workspace%27": { + "method": "GET", + "status_code": 200, + "json": { + "@odata.count": 3, + "value": [ + { + "id": "64ED5CAD-7C10-4684-8180-826122881108", + "isReadOnly": True, + "name": "demo-workspace", + "type": "Workspace", + } + ], + }, + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": { + "method": "GET", + "status_code": 200, + "json": {"value": []}, + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/datasources": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "datasourceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", + "datasourceType": "PostgreSql", + "connectionDetails": { + "database": "library_db", + "server": "foo", + }, + }, + ] + }, + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/executeQueries": { + "method": "POST", + "status_code": 200, + "json": execute_queries_response, + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/4674efd1-603c-4129-8d82-03cf2be05aff": { + "method": "GET", + "status_code": 200, + "json": { + "status": "SUCCEEDED", + }, + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/a674efd1-603c-4129-8d82-03cf2be05aff": { + "method": "GET", + "status_code": 200, + "json": { + "status": "SUCCEEDED", + }, + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4674efd1-603c-4129-8d82-03cf2be05aff": { + "method": "GET", + "status_code": 200, + "json": { + "workspaces": [ + { + "id": "64ED5CAD-7C10-4684-8180-826122881108", + "name": "demo-workspace", + "state": "Active", + "datasets": [ + { + "id": "05169CD2-E713-41E6-9600-1D8066D95445", + "endorsementDetails": {"endorsement": "Promoted"}, + "name": "test_sf_pbi_test", + "tables": [ + { + "name": "articles", + "source": [ + { + "expression": 'let\n Source = PostgreSQL.Database("localhost" , "mics" ),\n public_order_date = Source{[Schema="public",Item="order_date"]}[Data] \n in \n public_order_date', + } + ], + "datasourceUsages": [ + { + "datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", + } + ], + "columns": [ + { + "name": "link", + "description": "column description", + "dataType": "String", + "columnType": "DATA", + "isHidden": False, + }, + { + "name": "description", + "description": "column description", + "dataType": "String", + "columnType": "DATA", + "isHidden": False, + }, + { + "name": "topic", + "description": "column description", + "dataType": "String", + "columnType": "DATA", + "isHidden": False, + }, + ], + "measures": [ + { + "name": "view_count", + "description": "column description", + "expression": "let\n x", + "isHidden": False, + } + ], + }, + ], + }, + ], + "dashboards": [], + "reports": [], + }, + ] + }, + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/getInfo": { + "method": "POST", + "status_code": 200, + "json": scan_init_response, + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445": { + "method": "GET", + "status_code": 200, + "json": { + "id": "05169CD2-E713-41E6-9600-1D8066D95445", + "name": "library-dataset", + "description": "Library Dataset", + "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445", + }, + }, + } + + api_vs_response.update(override_data) + + for url in api_vs_response.keys(): + request_mock.register_uri( + api_vs_response[url]["method"], + url, + json=api_vs_response[url]["json"], + status_code=api_vs_response[url]["status_code"], + ) + + +def enable_logging(): + # set logging to console + logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) + logging.getLogger().setLevel(logging.DEBUG) + + +def mock_msal_cca(*args, **kwargs): + class MsalClient: + def acquire_token_for_client(self, *args, **kwargs): + return { + "access_token": "dummy", + } + + return MsalClient() + + +def default_source_config(): + return { + "client_id": "foo", + "client_secret": "bar", + "tenant_id": "0B0C960B-FCDF-4D0F-8C45-2E03BB59DDEB", + "workspace_id": "64ED5CAD-7C10-4684-8180-826122881108", + "extract_lineage": True, + "extract_reports": False, + "admin_apis_only": False, + "extract_ownership": True, + "convert_lineage_urns_to_lowercase": False, + "extract_independent_datasets": True, + "workspace_id_pattern": {"allow": ["64ED5CAD-7C10-4684-8180-826122881108"]}, + "extract_workspaces_to_containers": False, + "profiling": { + "enabled": True, + }, + "profile_pattern": {"allow": [".*"]}, + } + + +@freeze_time(FROZEN_TIME) +@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) +def test_profiling(mock_msal, pytestconfig, tmp_path, mock_time, requests_mock): + enable_logging() + + test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" + + register_mock_admin_api(request_mock=requests_mock) + + pipeline = Pipeline.create( + { + "run_id": "powerbi-test", + "source": { + "type": "powerbi", + "config": { + **default_source_config(), + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/powerbi_profiling.json", + }, + }, + } + ) + + pipeline.run() + pipeline.raise_from_status() + golden_file = "golden_test_profiling.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=f"{tmp_path}/powerbi_profiling.json", + golden_path=f"{test_resources_dir}/{golden_file}", + ) diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json index 6917b833d05a3..089c80f5c4ee7 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json @@ -89,10 +89,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -100,7 +101,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327546106, + "lastObserved": 1720073671135, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -319,10 +320,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -330,7 +332,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327546129, + "lastObserved": 1720073671192, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -469,10 +471,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -480,7 +483,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327546135, + "lastObserved": 1720073671209, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1217,10 +1220,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1228,7 +1232,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327546161, + "lastObserved": 1720073671300, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1414,10 +1418,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1425,7 +1430,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327546168, + "lastObserved": 1720073671321, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json index 5b47c09a79143..f285fc17a6fb4 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json @@ -87,10 +87,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -98,7 +99,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327442987, + "lastObserved": 1720073670616, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -303,10 +304,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -314,7 +316,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327443016, + "lastObserved": 1720073670661, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -432,10 +434,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -443,7 +446,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327443023, + "lastObserved": 1720073670669, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1113,10 +1116,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1124,7 +1128,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327443058, + "lastObserved": 1720073670731, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1289,10 +1293,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1300,7 +1305,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327443073, + "lastObserved": 1720073670744, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py index 818bc5198a5b1..ee1aafb6cf32d 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py +++ b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py @@ -204,8 +204,8 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", "status": "active", "subject": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", - "name": "Shubham jagtap", - "email": "Shubham.Jagtap@gslab.com", + "name": "john doe", + "email": "john.doe@example.com", "roles": [ "TenantAdmin", "AnalyticsAdmin", diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json index a81e6e927ef6a..12bb7734f30a6 100644 --- a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json @@ -39,6 +39,7 @@ "aspect": { "json": { "customProperties": { + "datasetId": "8891fd40-5470-4ff2-a74f-6e61ee44d3fc", "path": "Acryl Data" }, "externalUrl": "https://app.sigmacomputing.com/acryldata/b/49HFLTr6xytgrPly3PFsNC", @@ -186,6 +187,7 @@ "aspect": { "json": { "customProperties": { + "datasetId": "bd6b86e8-cd4a-4b25-ab65-f258c2a68a8f", "path": "Acryl Data/New Folder" }, "externalUrl": "https://app.sigmacomputing.com/acryldata/b/5LqGLu14qUnqh3cN6wRJBd", @@ -618,7 +620,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496054, + "lastObserved": 1718348049212, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1088,7 +1090,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496115, + "lastObserved": 1718348049268, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1239,7 +1241,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496188, + "lastObserved": 1718348049351, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1913,29 +1915,38 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:Warning", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", "changeType": "UPSERT", - "aspectName": "tagKey", + "aspectName": "upstreamLineage", "aspect": { "json": { - "name": "Warning" + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.pets,DEV)", + "type": "COPY" + } + ] } }, "systemMetadata": { - "lastObserved": 1713794496203, + "lastObserved": 1718348049380, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "tag", - "entityUrn": "urn:li:tag:Deprecated", + "entityUrn": "urn:li:tag:Warning", "changeType": "UPSERT", "aspectName": "tagKey", "aspect": { "json": { - "name": "Deprecated" + "name": "Warning" } }, "systemMetadata": { @@ -1945,26 +1956,17 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "entityType": "tag", + "entityUrn": "urn:li:tag:Deprecated", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "tagKey", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.pets,DEV)", - "type": "COPY" - } - ] + "name": "Deprecated" } }, "systemMetadata": { - "lastObserved": 1713794496202, + "lastObserved": 1713794496203, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_extract_lineage.json b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_extract_lineage.json new file mode 100644 index 0000000000000..4a1392accc9ad --- /dev/null +++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_extract_lineage.json @@ -0,0 +1,1861 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718003275457, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "8891fd40-5470-4ff2-a74f-6e61ee44d3fc", + "path": "Acryl Data" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/b/49HFLTr6xytgrPly3PFsNC", + "name": "PETS", + "qualifiedName": "PETS", + "description": "", + "created": { + "time": 1713188592664 + }, + "lastModified": { + "time": 1713188592664 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1718003275458, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + } + }, + "systemMetadata": { + "lastObserved": 1718003275459, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1718003275460, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275461, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275462, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718003275479, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "bd6b86e8-cd4a-4b25-ab65-f258c2a68a8f", + "path": "Acryl Data/New Folder" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/b/5LqGLu14qUnqh3cN6wRJBd", + "name": "PET_PROFILES_JOINED_DYNAMIC", + "qualifiedName": "PET_PROFILES_JOINED_DYNAMIC", + "description": "", + "created": { + "time": 1713189068019 + }, + "lastModified": { + "time": 1713189068019 + }, + "tags": [ + "Deprecated" + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275480, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + } + }, + "systemMetadata": { + "lastObserved": 1718003275481, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1718003275481, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275482, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Deprecated" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275482, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "New Folder" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275483, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "sigma", + "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", + "path": "Acryl Data", + "latestVersion": "2" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", + "name": "Acryl Workbook", + "created": { + "time": 1713188691477 + }, + "lastModified": { + "time": 1713189117302 + } + } + }, + "systemMetadata": { + "lastObserved": 1718003275484, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718003275485, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sigma" + } + }, + "systemMetadata": { + "lastObserved": 1718003275485, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275486, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1718003275486, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Warning" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275487, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + } + }, + "systemMetadata": { + "lastObserved": 1718003275487, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275488, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718003275489, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "ElementsCount": "2" + }, + "title": "Page 1", + "description": "", + "charts": [ + "urn:li:chart:(sigma,kH0MeihtGs)", + "urn:li:chart:(sigma,Ml9C5ezT5W)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1718003275489, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + }, + "systemMetadata": { + "lastObserved": 1718003275490, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275491, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718003275520, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "VizualizationType": "levelTable", + "type": "table" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=kH0MeihtGs&:fullScreen=true", + "title": "ADOPTIONS", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.adoptions,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275521, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + }, + "systemMetadata": { + "lastObserved": 1718003275522, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275522, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275525, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718003275582, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "VizualizationType": "bar", + "type": "visualization" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=Ml9C5ezT5W&:fullScreen=true", + "title": "Count of Profile Id by Status", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275583, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + }, + "systemMetadata": { + "lastObserved": 1718003275584, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Count of Profile Id)", + "schemaField": { + "fieldPath": "Count of Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275584, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275586, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Count of Profile Id)", + "schemaField": { + "fieldPath": "Count of Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275587, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718003275592, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "ElementsCount": "1" + }, + "title": "Page 2", + "description": "", + "charts": [ + "urn:li:chart:(sigma,tQJu5N1l81)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1718003275593, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + }, + "systemMetadata": { + "lastObserved": 1718003275594, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275594, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718003275654, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "VizualizationType": "levelTable", + "type": "table" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=tQJu5N1l81&:fullScreen=true", + "title": "PETS ADOPTIONS JOIN", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.adoptions,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275655, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + }, + "systemMetadata": { + "lastObserved": 1718003275656, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pk %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Pk (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Status %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Status (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Created At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Created At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Updated At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Updated At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275656, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275660, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pk %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Pk (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Status %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Status (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Created At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Created At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Updated At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Updated At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275661, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "sigma", + "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b" + }, + "name": "Acryl Data", + "created": { + "time": 1710232264826 + }, + "lastModified": { + "time": 1710232264826 + } + } + }, + "systemMetadata": { + "lastObserved": 1718003275665, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718003275666, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sigma" + } + }, + "systemMetadata": { + "lastObserved": 1718003275666, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275667, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1718003275667, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1718003275668, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.pets,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718003275668, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Deprecated", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Deprecated" + } + }, + "systemMetadata": { + "lastObserved": 1718003275669, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Warning", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Warning" + } + }, + "systemMetadata": { + "lastObserved": 1718003275670, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json index 50865ab0ff72e..f800cb19f8811 100644 --- a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json +++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json @@ -23,6 +23,7 @@ "aspect": { "json": { "customProperties": { + "datasetId": "8891fd40-5470-4ff2-a74f-6e61ee44d3fc", "path": "Acryl Data" }, "externalUrl": "https://app.sigmacomputing.com/acryldata/b/49HFLTr6xytgrPly3PFsNC", @@ -39,7 +40,7 @@ } }, "systemMetadata": { - "lastObserved": 1713795619219, + "lastObserved": 1718005767888, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -312,6 +313,7 @@ "aspect": { "json": { "customProperties": { + "datasetId": "bd6b86e8-cd4a-4b25-ab65-f258c2a68a8f", "path": "Acryl Data/New Folder" }, "externalUrl": "https://app.sigmacomputing.com/acryldata/b/5LqGLu14qUnqh3cN6wRJBd", @@ -330,7 +332,7 @@ } }, "systemMetadata": { - "lastObserved": 1713795619223, + "lastObserved": 1718005767900, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -844,7 +846,7 @@ } }, "systemMetadata": { - "lastObserved": 1713795619372, + "lastObserved": 1718348048900, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1172,7 +1174,7 @@ } }, "systemMetadata": { - "lastObserved": 1713795619447, + "lastObserved": 1718348048975, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1612,32 +1614,7 @@ } }, "systemMetadata": { - "lastObserved": 1713795619266, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.pets,PROD)", - "type": "COPY" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1713795619462, + "lastObserved": 1718348048836, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1824,6 +1801,31 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.pets,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718348048990, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:Deprecated", diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json new file mode 100644 index 0000000000000..d6b702bdfd669 --- /dev/null +++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json @@ -0,0 +1,1821 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718004101661, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "8891fd40-5470-4ff2-a74f-6e61ee44d3fc", + "path": "Acryl Data" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/b/49HFLTr6xytgrPly3PFsNC", + "name": "PETS", + "qualifiedName": "PETS", + "description": "", + "created": { + "time": 1713188592664 + }, + "lastModified": { + "time": 1713188592664 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1718004101662, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + } + }, + "systemMetadata": { + "lastObserved": 1718004101663, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1718004101663, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101664, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101665, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718004101675, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "bd6b86e8-cd4a-4b25-ab65-f258c2a68a8f", + "path": "Acryl Data/New Folder" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/b/5LqGLu14qUnqh3cN6wRJBd", + "name": "PET_PROFILES_JOINED_DYNAMIC", + "qualifiedName": "PET_PROFILES_JOINED_DYNAMIC", + "description": "", + "created": { + "time": 1713189068019 + }, + "lastModified": { + "time": 1713189068019 + }, + "tags": [ + "Deprecated" + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101675, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + } + }, + "systemMetadata": { + "lastObserved": 1718004101676, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1718004101677, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101677, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Deprecated" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101678, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "New Folder" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101678, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "sigma", + "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", + "path": "New Acryl Data", + "latestVersion": "2" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", + "name": "Acryl Workbook", + "created": { + "time": 1713188691477 + }, + "lastModified": { + "time": 1713189117302 + } + } + }, + "systemMetadata": { + "lastObserved": 1718004101680, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718004101680, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sigma" + } + }, + "systemMetadata": { + "lastObserved": 1718004101681, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101681, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1718004101682, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Warning" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101683, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b83da80a4d444484521d9f7aca958742" + } + }, + "systemMetadata": { + "lastObserved": 1718004101683, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", + "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101684, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718004101684, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "ElementsCount": "2" + }, + "title": "Page 1", + "description": "", + "charts": [ + "urn:li:chart:(sigma,kH0MeihtGs)", + "urn:li:chart:(sigma,Ml9C5ezT5W)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1718004101685, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + }, + "systemMetadata": { + "lastObserved": 1718004101686, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", + "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" + }, + { + "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101686, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718004101687, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "VizualizationType": "levelTable", + "type": "table" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=kH0MeihtGs&:fullScreen=true", + "title": "ADOPTIONS", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [] + } + }, + "systemMetadata": { + "lastObserved": 1718004101688, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + }, + "systemMetadata": { + "lastObserved": 1718004101689, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101689, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", + "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" + }, + { + "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101692, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718004101693, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "VizualizationType": "bar", + "type": "visualization" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=Ml9C5ezT5W&:fullScreen=true", + "title": "Count of Profile Id by Status", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [] + } + }, + "systemMetadata": { + "lastObserved": 1718004101694, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + }, + "systemMetadata": { + "lastObserved": 1718004101695, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Count of Profile Id)", + "schemaField": { + "fieldPath": "Count of Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101695, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", + "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" + }, + { + "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101697, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Count of Profile Id)", + "schemaField": { + "fieldPath": "Count of Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101698, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718004101703, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "ElementsCount": "1" + }, + "title": "Page 2", + "description": "", + "charts": [ + "urn:li:chart:(sigma,tQJu5N1l81)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1718004101703, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + }, + "systemMetadata": { + "lastObserved": 1718004101704, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", + "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" + }, + { + "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101704, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718004101706, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "VizualizationType": "levelTable", + "type": "table" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=tQJu5N1l81&:fullScreen=true", + "title": "PETS ADOPTIONS JOIN", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [] + } + }, + "systemMetadata": { + "lastObserved": 1718004101707, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + }, + "systemMetadata": { + "lastObserved": 1718004101708, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pk %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Pk (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Status %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Status (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Created At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Created At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Updated At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Updated At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101708, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", + "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" + }, + { + "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101712, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pk %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Pk (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Status %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Status (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Created At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Created At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Updated At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Updated At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101713, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "sigma", + "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b" + }, + "name": "Acryl Data", + "created": { + "time": 1710232264826 + }, + "lastModified": { + "time": 1710232264826 + } + } + }, + "systemMetadata": { + "lastObserved": 1718004101717, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1718004101718, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sigma" + } + }, + "systemMetadata": { + "lastObserved": 1718004101718, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1718004101719, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1718004101719, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1718004101720, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Deprecated", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Deprecated" + } + }, + "systemMetadata": { + "lastObserved": 1718004101720, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Warning", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Warning" + } + }, + "systemMetadata": { + "lastObserved": 1718004101721, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sigma/test_sigma.py b/metadata-ingestion/tests/integration/sigma/test_sigma.py index c9a845e59ddf2..b6e9db99eed39 100644 --- a/metadata-ingestion/tests/integration/sigma/test_sigma.py +++ b/metadata-ingestion/tests/integration/sigma/test_sigma.py @@ -13,11 +13,30 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "status_code": 200, "json": { "access_token": "717de8281754fe8e302b1ee69f1c9553faf0331cabd8712f459c", + "refresh_token": "124de8281754fe8e302b1ee69f1c9553faf0331cabd8712f442v", "token_type": "bearer", "expires_in": 3599, }, }, - "https://aws-api.sigmacomputing.com/v2/files": { + "https://aws-api.sigmacomputing.com/v2/workspaces?limit=50": { + "method": "GET", + "status_code": 200, + "json": { + "entries": [ + { + "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b", + "name": "Acryl Data", + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2024-03-12T08:31:04.826Z", + "updatedAt": "2024-03-12T08:31:04.826Z", + }, + ], + "total": 1, + "nextPage": None, + }, + }, + "https://aws-api.sigmacomputing.com/v2/files?typeFilters=dataset": { "method": "GET", "status_code": 200, "json": { @@ -54,22 +73,16 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "updatedAt": "2024-04-15T13:51:08.019Z", "isArchived": False, }, - { - "id": "1b47afdb-db4e-4a2c-9fa4-fc1332f4a097", - "urlId": "Ptyl1jrKEO18RDX9y1d4P", - "name": "New Folder", - "type": "folder", - "parentId": "3ee61405-3be2-4000-ba72-60d36757b95b", - "parentUrlId": "1UGFyEQCHqwPfQoAec3xJ9", - "permission": "edit", - "path": "Acryl Data", - "badge": None, - "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", - "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", - "createdAt": "2024-04-15T13:35:39.512Z", - "updatedAt": "2024-04-15T13:35:39.512Z", - "isArchived": False, - }, + ], + "total": 2, + "nextPage": None, + }, + }, + "https://aws-api.sigmacomputing.com/v2/files?typeFilters=workbook": { + "method": "GET", + "status_code": 200, + "json": { + "entries": [ { "id": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", "urlId": "4JRFW1HThPI1K3YTjouXI7", @@ -87,35 +100,42 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "isArchived": False, }, ], - "total": 4, + "total": 1, "nextPage": None, }, }, - "https://aws-api.sigmacomputing.com/v2/workspaces/3ee61405-3be2-4000-ba72-60d36757b95b": { - "method": "GET", - "status_code": 200, - "json": { - "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b", - "name": "Acryl Data", - "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", - "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", - "createdAt": "2024-03-12T08:31:04.826Z", - "updatedAt": "2024-03-12T08:31:04.826Z", - }, - }, - "https://aws-api.sigmacomputing.com/v2/datasets/8891fd40-5470-4ff2-a74f-6e61ee44d3fc": { + "https://aws-api.sigmacomputing.com/v2/datasets": { "method": "GET", "status_code": 200, "json": { - "datasetId": "8891fd40-5470-4ff2-a74f-6e61ee44d3fc", - "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", - "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", - "createdAt": "2024-04-15T13:43:12.664Z", - "updatedAt": "2024-04-15T13:43:12.664Z", - "name": "PETS", - "description": "", - "url": "https://app.sigmacomputing.com/acryldata/b/49HFLTr6xytgrPly3PFsNC", - "isArchived": False, + "entries": [ + { + "datasetId": "8891fd40-5470-4ff2-a74f-6e61ee44d3fc", + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2024-04-15T13:43:12.664Z", + "updatedAt": "2024-04-15T13:43:12.664Z", + "name": "PETS", + "description": "", + "url": "https://app.sigmacomputing.com/acryldata/b/49HFLTr6xytgrPly3PFsNC", + "isArchived": False, + }, + { + "datasetId": "bd6b86e8-cd4a-4b25-ab65-f258c2a68a8f", + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2024-04-15T13:51:08.019Z", + "updatedAt": "2024-04-15T13:51:08.019Z", + "name": "PET_PROFILES_JOINED_DYNAMIC", + "description": "", + "url": "https://app.sigmacomputing.com/acryldata/b/5LqGLu14qUnqh3cN6wRJBd", + "isArchived": False, + "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b", + "path": "Acryl Data/New Folder", + }, + ], + "total": 2, + "nextPage": None, }, }, "https://aws-api.sigmacomputing.com/v2/files/1b47afdb-db4e-4a2c-9fa4-fc1332f4a097": { @@ -138,39 +158,28 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "isArchived": False, }, }, - "https://aws-api.sigmacomputing.com/v2/datasets/bd6b86e8-cd4a-4b25-ab65-f258c2a68a8f": { + "https://aws-api.sigmacomputing.com/v2/workbooks": { "method": "GET", "status_code": 200, "json": { - "datasetId": "bd6b86e8-cd4a-4b25-ab65-f258c2a68a8f", - "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", - "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", - "createdAt": "2024-04-15T13:51:08.019Z", - "updatedAt": "2024-04-15T13:51:08.019Z", - "name": "PET_PROFILES_JOINED_DYNAMIC", - "description": "", - "url": "https://app.sigmacomputing.com/acryldata/b/5LqGLu14qUnqh3cN6wRJBd", - "isArchived": False, - "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b", - "path": "Acryl Data/New Folder", - }, - }, - "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b": { - "method": "GET", - "status_code": 200, - "json": { - "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", - "workbookUrlId": "4JRFW1HThPI1K3YTjouXI7", - "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", - "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", - "createdAt": "2024-04-15T13:44:51.477Z", - "updatedAt": "2024-04-15T13:51:57.302Z", - "name": "Acryl Workbook", - "url": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", - "path": "Acryl Data", - "latestVersion": 2, - "isArchived": False, - "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b", + "entries": [ + { + "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", + "workbookUrlId": "4JRFW1HThPI1K3YTjouXI7", + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2024-04-15T13:44:51.477Z", + "updatedAt": "2024-04-15T13:51:57.302Z", + "name": "Acryl Workbook", + "url": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", + "path": "Acryl Data", + "latestVersion": 2, + "isArchived": False, + "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b", + }, + ], + "total": 1, + "nextPage": None, }, }, "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/pages": { @@ -255,7 +264,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "json": { "elementId": "kH0MeihtGs", "name": "ADOPTIONS", - "sql": 'select PK "Pk", PET_FK "Pet Fk", HUMAN_FK "Human Fk", STATUS "Status", CAST_TIMESTAMP_TO_DATETIME_7 "Created At", CAST_TIMESTAMP_TO_DATETIME_8 "Updated At" from (select PK, PET_FK, HUMAN_FK, STATUS, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_7, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_8 from (select * from LONG_TAIL_COMPANIONS.ADOPTION.ADOPTIONS ADOPTIONS limit 1000) Q1) Q2 limit 1000\n\n-- Sigma Σ {"request-id":"3d4bf15e-6a17-4967-ad2a-213341233a04","email":"Shubham.Jagtap@gslab.com"}', + "sql": 'select PK "Pk", PET_FK "Pet Fk", HUMAN_FK "Human Fk", STATUS "Status", CAST_TIMESTAMP_TO_DATETIME_7 "Created At", CAST_TIMESTAMP_TO_DATETIME_8 "Updated At" from (select PK, PET_FK, HUMAN_FK, STATUS, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_7, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_8 from (select * from LONG_TAIL_COMPANIONS.ADOPTION.ADOPTIONS ADOPTIONS limit 1000) Q1) Q2 limit 1000\n\n-- Sigma Σ {"request-id":"3d4bf15e-6a17-4967-ad2a-213341233a04","email":"john.doe@example.com"}', }, }, "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/lineage/elements/Ml9C5ezT5W": { @@ -290,7 +299,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "json": { "elementId": "Ml9C5ezT5W", "name": "Count of Profile Id by Status", - "sql": 'with Q1 as (select PK, PROFILE_ID, STATUS, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_6, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_7 from LONG_TAIL_COMPANIONS.ADOPTION.PETS PETS) select STATUS_10 "Status", COUNT_23 "Count of Profile Id", PK_8 "Pk", PROFILE_ID_9 "Profile Id", CAST_TIMESTAMP_TO_DATETIME_8 "Created At", CAST_TIMESTAMP_TO_DATETIME_9 "Updated At" from (select Q3.PK_8 PK_8, Q3.PROFILE_ID_9 PROFILE_ID_9, Q3.STATUS_10 STATUS_10, Q3.CAST_TIMESTAMP_TO_DATETIME_8 CAST_TIMESTAMP_TO_DATETIME_8, Q3.CAST_TIMESTAMP_TO_DATETIME_9 CAST_TIMESTAMP_TO_DATETIME_9, Q6.COUNT_23 COUNT_23, Q6.STATUS_11 STATUS_11 from (select PK PK_8, PROFILE_ID PROFILE_ID_9, STATUS STATUS_10, CAST_TIMESTAMP_TO_DATETIME_6 CAST_TIMESTAMP_TO_DATETIME_8, CAST_TIMESTAMP_TO_DATETIME_7 CAST_TIMESTAMP_TO_DATETIME_9 from Q1 Q2 order by STATUS_10 asc limit 1000) Q3 left join (select count(PROFILE_ID_9) COUNT_23, STATUS_10 STATUS_11 from (select PROFILE_ID PROFILE_ID_9, STATUS STATUS_10 from Q1 Q4) Q5 group by STATUS_10) Q6 on equal_null(Q3.STATUS_10, Q6.STATUS_11)) Q8 order by STATUS_10 asc limit 1000\n\n-- Sigma Σ {"request-id":"988dd6b5-0678-4421-ae14-21594c0ee97a","email":"Shubham.Jagtap@gslab.com"}', + "sql": 'with Q1 as (select PK, PROFILE_ID, STATUS, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_6, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_7 from LONG_TAIL_COMPANIONS.ADOPTION.PETS PETS) select STATUS_10 "Status", COUNT_23 "Count of Profile Id", PK_8 "Pk", PROFILE_ID_9 "Profile Id", CAST_TIMESTAMP_TO_DATETIME_8 "Created At", CAST_TIMESTAMP_TO_DATETIME_9 "Updated At" from (select Q3.PK_8 PK_8, Q3.PROFILE_ID_9 PROFILE_ID_9, Q3.STATUS_10 STATUS_10, Q3.CAST_TIMESTAMP_TO_DATETIME_8 CAST_TIMESTAMP_TO_DATETIME_8, Q3.CAST_TIMESTAMP_TO_DATETIME_9 CAST_TIMESTAMP_TO_DATETIME_9, Q6.COUNT_23 COUNT_23, Q6.STATUS_11 STATUS_11 from (select PK PK_8, PROFILE_ID PROFILE_ID_9, STATUS STATUS_10, CAST_TIMESTAMP_TO_DATETIME_6 CAST_TIMESTAMP_TO_DATETIME_8, CAST_TIMESTAMP_TO_DATETIME_7 CAST_TIMESTAMP_TO_DATETIME_9 from Q1 Q2 order by STATUS_10 asc limit 1000) Q3 left join (select count(PROFILE_ID_9) COUNT_23, STATUS_10 STATUS_11 from (select PROFILE_ID PROFILE_ID_9, STATUS STATUS_10 from Q1 Q4) Q5 group by STATUS_10) Q6 on equal_null(Q3.STATUS_10, Q6.STATUS_11)) Q8 order by STATUS_10 asc limit 1000\n\n-- Sigma Σ {"request-id":"988dd6b5-0678-4421-ae14-21594c0ee97a","email":"john.doe@example.com"}', }, }, "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/pages/DFSieiAcgo/elements": { @@ -366,29 +375,31 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "json": { "elementId": "tQJu5N1l81", "name": "PETS ADOPTIONS JOIN", - "sql": 'select PK_8 "Pk", PROFILE_ID_9 "Profile Id", STATUS_10 "Status", CAST_TIMESTAMP_TO_DATETIME_11 "Created At", CAST_TIMESTAMP_TO_DATETIME_12 "Updated At", PK_13 "Pk (ADOPTIONS)", PET_FK_14 "Pet Fk", HUMAN_FK_15 "Human Fk", STATUS_16 "Status (ADOPTIONS)", CAST_TIMESTAMP_TO_DATETIME_19 "Created At (ADOPTIONS)", CAST_TIMESTAMP_TO_DATETIME_20 "Updated At (ADOPTIONS)" from (select PK_8, PROFILE_ID_9, STATUS_10, CAST_TIMESTAMP_TO_DATETIME_11, CAST_TIMESTAMP_TO_DATETIME_12, PK_13, PET_FK_14, HUMAN_FK_15, STATUS_16, CREATED_AT_17::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_19, UPDATED_AT_18::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_20 from (select Q1.PK_8 PK_8, Q1.PROFILE_ID_9 PROFILE_ID_9, Q1.STATUS_10 STATUS_10, Q1.CAST_TIMESTAMP_TO_DATETIME_11 CAST_TIMESTAMP_TO_DATETIME_11, Q1.CAST_TIMESTAMP_TO_DATETIME_12 CAST_TIMESTAMP_TO_DATETIME_12, Q2.PK PK_13, Q2.PET_FK PET_FK_14, Q2.HUMAN_FK HUMAN_FK_15, Q2.STATUS STATUS_16, Q2.CREATED_AT CREATED_AT_17, Q2.UPDATED_AT UPDATED_AT_18 from (select PK PK_8, PROFILE_ID PROFILE_ID_9, STATUS STATUS_10, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_11, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_12 from LONG_TAIL_COMPANIONS.ADOPTION.PETS PETS) Q1 inner join LONG_TAIL_COMPANIONS.ADOPTION.ADOPTIONS Q2 on (Q1.PK_8 = Q2.PET_FK) limit 1000) Q4) Q5 limit 1000\n\n-- Sigma Σ {"request-id":"f5a997ef-b80c-47f1-b32e-9cd0f50cd491","email":"Shubham.Jagtap@gslab.com"}', + "sql": 'select PK_8 "Pk", PROFILE_ID_9 "Profile Id", STATUS_10 "Status", CAST_TIMESTAMP_TO_DATETIME_11 "Created At", CAST_TIMESTAMP_TO_DATETIME_12 "Updated At", PK_13 "Pk (ADOPTIONS)", PET_FK_14 "Pet Fk", HUMAN_FK_15 "Human Fk", STATUS_16 "Status (ADOPTIONS)", CAST_TIMESTAMP_TO_DATETIME_19 "Created At (ADOPTIONS)", CAST_TIMESTAMP_TO_DATETIME_20 "Updated At (ADOPTIONS)" from (select PK_8, PROFILE_ID_9, STATUS_10, CAST_TIMESTAMP_TO_DATETIME_11, CAST_TIMESTAMP_TO_DATETIME_12, PK_13, PET_FK_14, HUMAN_FK_15, STATUS_16, CREATED_AT_17::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_19, UPDATED_AT_18::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_20 from (select Q1.PK_8 PK_8, Q1.PROFILE_ID_9 PROFILE_ID_9, Q1.STATUS_10 STATUS_10, Q1.CAST_TIMESTAMP_TO_DATETIME_11 CAST_TIMESTAMP_TO_DATETIME_11, Q1.CAST_TIMESTAMP_TO_DATETIME_12 CAST_TIMESTAMP_TO_DATETIME_12, Q2.PK PK_13, Q2.PET_FK PET_FK_14, Q2.HUMAN_FK HUMAN_FK_15, Q2.STATUS STATUS_16, Q2.CREATED_AT CREATED_AT_17, Q2.UPDATED_AT UPDATED_AT_18 from (select PK PK_8, PROFILE_ID PROFILE_ID_9, STATUS STATUS_10, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_11, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_12 from LONG_TAIL_COMPANIONS.ADOPTION.PETS PETS) Q1 inner join LONG_TAIL_COMPANIONS.ADOPTION.ADOPTIONS Q2 on (Q1.PK_8 = Q2.PET_FK) limit 1000) Q4) Q5 limit 1000\n\n-- Sigma Σ {"request-id":"f5a997ef-b80c-47f1-b32e-9cd0f50cd491","email":"john.doe@example.com"}', }, }, - "https://aws-api.sigmacomputing.com/v2/members/CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1": { + "https://aws-api.sigmacomputing.com/v2/members": { "method": "GET", "status_code": 200, - "json": { - "organizationId": "b94da709-176c-4242-bea6-6760f34c9228", - "memberId": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", - "memberType": "admin", - "firstName": "Shubham", - "lastName": "Jagtap", - "email": "Shubham.Jagtap@gslab.com", - "profileImgUrl": None, - "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", - "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", - "createdAt": "2023-11-28T10:59:20.957Z", - "updatedAt": "2024-03-12T21:21:17.996Z", - "homeFolderId": "9bb94df1-e8af-49eb-9c37-2bd40b0efb2e", - "userKind": "internal", - "isArchived": False, - "isInactive": False, - }, + "json": [ + { + "organizationId": "b94da709-176c-4242-bea6-6760f34c9228", + "memberId": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "memberType": "admin", + "firstName": "Shubham", + "lastName": "Jagtap", + "email": "john.doe@example.com", + "profileImgUrl": None, + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2023-11-28T10:59:20.957Z", + "updatedAt": "2024-03-12T21:21:17.996Z", + "homeFolderId": "9bb94df1-e8af-49eb-9c37-2bd40b0efb2e", + "userKind": "internal", + "isArchived": False, + "isInactive": False, + }, + ], }, } @@ -491,3 +502,106 @@ def test_platform_instance_ingest(pytestconfig, tmp_path, requests_mock): output_path=output_path, golden_path=f"{test_resources_dir}/{golden_file}", ) + + +@pytest.mark.integration +def test_sigma_ingest_shared_entities(pytestconfig, tmp_path, requests_mock): + + test_resources_dir = pytestconfig.rootpath / "tests/integration/sigma" + + override_data = { + "https://aws-api.sigmacomputing.com/v2/workbooks": { + "method": "GET", + "status_code": 200, + "json": { + "entries": [ + { + "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", + "workbookUrlId": "4JRFW1HThPI1K3YTjouXI7", + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2024-04-15T13:44:51.477Z", + "updatedAt": "2024-04-15T13:51:57.302Z", + "name": "Acryl Workbook", + "url": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", + "path": "New Acryl Data", + "latestVersion": 2, + "isArchived": False, + "workspaceId": "4pe61405-3be2-4000-ba72-60d36757b95b", + }, + ], + "total": 1, + "nextPage": None, + }, + }, + "https://aws-api.sigmacomputing.com/v2/files?typeFilters=workbook": { + "method": "GET", + "status_code": 200, + "json": { + "entries": [ + { + "id": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", + "urlId": "4JRFW1HThPI1K3YTjouXI7", + "name": "Acryl Workbook", + "type": "workbook", + "parentId": "4pe61405-3be2-4000-ba72-60d36757b95b", + "parentUrlId": "1UGFyEQCHqwPfQoAec3xJ9", + "permission": "edit", + "path": "New Acryl Data", + "badge": "Warning", + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2024-04-15T13:44:51.477Z", + "updatedAt": "2024-04-15T13:51:57.302Z", + "isArchived": False, + }, + ], + "total": 1, + "nextPage": None, + }, + }, + "https://aws-api.sigmacomputing.com/v2/workspaces/4pe61405-3be2-4000-ba72-60d36757b95b": { + "method": "GET", + "status_code": 403, + "json": {}, + }, + } + + register_mock_api(request_mock=requests_mock, override_data=override_data) + + output_path: str = f"{tmp_path}/sigma_ingest_shared_entities_mces.json" + + pipeline = Pipeline.create( + { + "run_id": "sigma-test", + "source": { + "type": "sigma", + "config": { + "client_id": "CLIENTID", + "client_secret": "CLIENTSECRET", + "ingest_shared_entities": True, + "chart_sources_platform_mapping": { + "Acryl Data/Acryl Workbook": { + "data_source_platform": "snowflake" + }, + }, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": output_path, + }, + }, + } + ) + + pipeline.run() + pipeline.raise_from_status() + golden_file = "golden_test_sigma_ingest_shared_entities_mces.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=output_path, + golden_path=f"{test_resources_dir}/{golden_file}", + ) diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py index 30c4b2bec3a04..5ef2eb420b8ed 100644 --- a/metadata-ingestion/tests/integration/snowflake/common.py +++ b/metadata-ingestion/tests/integration/snowflake/common.py @@ -224,8 +224,6 @@ def default_query_results( # noqa: C901 ] elif query == SnowflakeQuery.tables_for_database("TEST_DB"): raise Exception("Information schema query returned too much data") - elif query == SnowflakeQuery.show_views_for_database("TEST_DB"): - raise Exception("Information schema query returned too much data") elif query == SnowflakeQuery.tables_for_schema("TEST_SCHEMA", "TEST_DB"): return [ { @@ -241,7 +239,8 @@ def default_query_results( # noqa: C901 } for tbl_idx in range(1, num_tables + 1) ] - elif query == SnowflakeQuery.show_views_for_schema("TEST_SCHEMA", "TEST_DB"): + elif query == SnowflakeQuery.show_views_for_database("TEST_DB"): + # TODO: Add tests for view pagination. return [ { "schema_name": "TEST_SCHEMA", @@ -253,26 +252,11 @@ def default_query_results( # noqa: C901 for view_idx in range(1, num_views + 1) ] elif query == SnowflakeQuery.columns_for_schema("TEST_SCHEMA", "TEST_DB"): - raise Exception("Information schema query returned too much data") - elif query in [ - *[ - SnowflakeQuery.columns_for_table( - f"TABLE_{tbl_idx}", "TEST_SCHEMA", "TEST_DB" - ) - for tbl_idx in range(1, num_tables + 1) - ], - *[ - SnowflakeQuery.columns_for_table( - f"VIEW_{view_idx}", "TEST_SCHEMA", "TEST_DB" - ) - for view_idx in range(1, num_views + 1) - ], - ]: return [ { - # "TABLE_CATALOG": "TEST_DB", - # "TABLE_SCHEMA": "TEST_SCHEMA", - # "TABLE_NAME": "TABLE_{}".format(tbl_idx), + "TABLE_CATALOG": "TEST_DB", + "TABLE_SCHEMA": "TEST_SCHEMA", + "TABLE_NAME": table_name, "COLUMN_NAME": f"COL_{col_idx}", "ORDINAL_POSITION": col_idx, "IS_NULLABLE": "NO", @@ -282,6 +266,10 @@ def default_query_results( # noqa: C901 "NUMERIC_PRECISION": None if col_idx > 1 else 38, "NUMERIC_SCALE": None if col_idx > 1 else 0, } + for table_name in ( + [f"TABLE_{tbl_idx}" for tbl_idx in range(1, num_tables + 1)] + + [f"VIEW_{view_idx}" for view_idx in range(1, num_views + 1)] + ) for col_idx in range(1, num_cols + 1) ] elif query in ( @@ -445,69 +433,6 @@ def default_query_results( # noqa: C901 for i in range(num_usages) ] return mock - elif query in ( - snowflake_query.SnowflakeQuery.table_to_table_lineage_history( - 1654473600000, - 1654586220000, - ), - snowflake_query.SnowflakeQuery.table_to_table_lineage_history( - 1654473600000, 1654586220000, False - ), - ): - return [ - { - "DOWNSTREAM_TABLE_NAME": f"TEST_DB.TEST_SCHEMA.TABLE_{op_idx}", - "UPSTREAM_TABLE_NAME": "TEST_DB.TEST_SCHEMA.TABLE_2", - "UPSTREAM_TABLE_COLUMNS": json.dumps( - [ - {"columnId": 0, "columnName": f"COL_{col_idx}"} - for col_idx in range(1, num_cols + 1) - ] - ), - "DOWNSTREAM_TABLE_COLUMNS": json.dumps( - [ - { - "columnId": 0, - "columnName": f"COL_{col_idx}", - "directSources": [ - { - "columnName": f"COL_{col_idx}", - "objectDomain": "Table", - "objectId": 0, - "objectName": "TEST_DB.TEST_SCHEMA.TABLE_2", - } - ], - } - for col_idx in range(1, num_cols + 1) - ] - ), - } - for op_idx in range(1, num_ops + 1) - ] + [ - { - "DOWNSTREAM_TABLE_NAME": "TEST_DB.TEST_SCHEMA.TABLE_1", - "UPSTREAM_TABLE_NAME": "OTHER_DB.OTHER_SCHEMA.TABLE_1", - "UPSTREAM_TABLE_COLUMNS": json.dumps( - [{"columnId": 0, "columnName": "COL_1"}] - ), - "DOWNSTREAM_TABLE_COLUMNS": json.dumps( - [ - { - "columnId": 0, - "columnName": "COL_1", - "directSources": [ - { - "columnName": "COL_1", - "objectDomain": "Table", - "objectId": 0, - "objectName": "OTHER_DB.OTHER_SCHEMA.TABLE_1", - } - ], - } - ] - ), - } - ] elif query in ( snowflake_query.SnowflakeQuery.table_to_table_lineage_history_v2( start_time_millis=1654473600000, @@ -603,7 +528,7 @@ def default_query_results( # noqa: C901 snowflake_query.SnowflakeQuery.table_to_table_lineage_history_v2( start_time_millis=1654473600000, end_time_millis=1654586220000, - include_view_lineage=False, + include_view_lineage=True, include_column_lineage=False, ), ): @@ -674,53 +599,15 @@ def default_query_results( # noqa: C901 ), } ] - elif query in [ - snowflake_query.SnowflakeQuery.view_lineage_history( - 1654473600000, - 1654586220000, - ), - snowflake_query.SnowflakeQuery.view_lineage_history( - 1654473600000, 1654586220000, False - ), - ]: - return [ - { - "DOWNSTREAM_TABLE_NAME": "TEST_DB.TEST_SCHEMA.TABLE_1", - "VIEW_NAME": "TEST_DB.TEST_SCHEMA.VIEW_1", - "VIEW_DOMAIN": "VIEW", - "VIEW_COLUMNS": json.dumps( - [ - {"columnId": 0, "columnName": f"COL_{col_idx}"} - for col_idx in range(1, num_cols + 1) - ] - ), - "DOWNSTREAM_TABLE_DOMAIN": "TABLE", - "DOWNSTREAM_TABLE_COLUMNS": json.dumps( - [ - { - "columnId": 0, - "columnName": f"COL_{col_idx}", - "directSources": [ - { - "columnName": f"COL_{col_idx}", - "objectDomain": "Table", - "objectId": 0, - "objectName": "TEST_DB.TEST_SCHEMA.TABLE_2", - } - ], - } - for col_idx in range(1, num_cols + 1) - ] - ), - } - ] elif query in [ snowflake_query.SnowflakeQuery.view_dependencies_v2(), snowflake_query.SnowflakeQuery.view_dependencies(), snowflake_query.SnowflakeQuery.show_external_tables(), snowflake_query.SnowflakeQuery.copy_lineage_history( - 1654473600000, - 1654586220000, + start_time_millis=1654473600000, end_time_millis=1654621200000 + ), + snowflake_query.SnowflakeQuery.copy_lineage_history( + start_time_millis=1654473600000, end_time_millis=1654586220000 ), ]: return [] diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json index 82b29c051114a..5cba4e8b33822 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json @@ -4513,9 +4513,6 @@ "aspect": { "json": { "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" - }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" }, @@ -4524,13 +4521,49 @@ }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,other_db.other_schema.table_1,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2024_07_09-14_45_18", "lastRunId": "no-run-id-provided" } }, @@ -5102,13 +5135,43 @@ }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2024_07_09-14_45_18", "lastRunId": "no-run-id-provided" } }, @@ -5136,18 +5199,48 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2024_07_09-14_45_18", "lastRunId": "no-run-id-provided" } }, @@ -5659,18 +5752,48 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2024_07_09-14_45_18", "lastRunId": "no-run-id-provided" } }, @@ -5862,18 +5985,48 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2024_07_09-14_45_18", "lastRunId": "no-run-id-provided" } }, @@ -6265,18 +6418,48 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2024_07_09-14_45_18", "lastRunId": "no-run-id-provided" } }, @@ -6674,18 +6857,48 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2024_07_09-14_45_18", "lastRunId": "no-run-id-provided" } }, @@ -6827,18 +7040,48 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2024_07_09-14_45_18", "lastRunId": "no-run-id-provided" } }, @@ -6850,18 +7093,78 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2024_07_09-14_45_18", "lastRunId": "no-run-id-provided" } }, @@ -6962,18 +7265,48 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2024_07_09-14_45_18", "lastRunId": "no-run-id-provided" } }, @@ -7099,18 +7432,48 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2024_07_09-14_45_18", "lastRunId": "no-run-id-provided" } }, @@ -7122,18 +7485,78 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2024_07_09-14_45_18", "lastRunId": "no-run-id-provided" } }, diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json index 78d3b920767f7..71a74f883bf1b 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json @@ -3892,11 +3892,71 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_10)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_8)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_10)" } ] } @@ -4111,11 +4171,71 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_10)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_8)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_10)" } ] } diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py index 81487d38eda7d..ca694b02cff01 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py @@ -119,6 +119,7 @@ def test_snowflake_basic(pytestconfig, tmp_path, mock_time, mock_datahub_graph): include_table_lineage=True, include_view_lineage=True, include_usage_stats=True, + format_sql_queries=True, validate_upstreams_against_patterns=False, include_operational_stats=True, email_as_user_identifier=True, @@ -176,11 +177,13 @@ def test_snowflake_basic(pytestconfig, tmp_path, mock_time, mock_datahub_graph): ], ) report = cast(SnowflakeV2Report, pipeline.source.get_report()) - assert report.lru_cache_info["get_tables_for_database"]["misses"] == 1 - assert report.lru_cache_info["get_views_for_database"]["misses"] == 1 - assert report.lru_cache_info["get_columns_for_schema"]["misses"] == 1 - assert report.lru_cache_info["get_pk_constraints_for_schema"]["misses"] == 1 - assert report.lru_cache_info["get_fk_constraints_for_schema"]["misses"] == 1 + assert report.data_dictionary_cache is not None + cache_info = report.data_dictionary_cache.as_obj() + assert cache_info["get_tables_for_database"]["misses"] == 1 + assert cache_info["get_views_for_database"]["misses"] == 1 + assert cache_info["get_columns_for_schema"]["misses"] == 1 + assert cache_info["get_pk_constraints_for_schema"]["misses"] == 1 + assert cache_info["get_fk_constraints_for_schema"]["misses"] == 1 @freeze_time(FROZEN_TIME) @@ -213,6 +216,7 @@ def test_snowflake_private_link(pytestconfig, tmp_path, mock_time, mock_datahub_ include_views=True, include_view_lineage=True, include_usage_stats=False, + format_sql_queries=True, incremental_lineage=False, include_operational_stats=False, platform_instance="instance1", diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py index 9760ea1a9c72b..0b838b0bb59c3 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py @@ -1,20 +1,17 @@ from datetime import datetime, timezone from unittest import mock +import pytest from freezegun import freeze_time from pytest import fixture from datahub.configuration.common import AllowDenyPattern, DynamicTypedConfig -from datahub.ingestion.run.pipeline import Pipeline +from datahub.ingestion.run.pipeline import Pipeline, PipelineInitError from datahub.ingestion.run.pipeline_config import PipelineConfig, SourceConfig from datahub.ingestion.source.snowflake import snowflake_query from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeV2Config from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery -from tests.integration.snowflake.common import ( - FROZEN_TIME, - NUM_TABLES, - default_query_results, -) +from tests.integration.snowflake.common import FROZEN_TIME, default_query_results def query_permission_error_override(fn, override_for_query, error_msg): @@ -76,9 +73,10 @@ def test_snowflake_missing_role_access_causes_pipeline_failure( "250001 (08001): Failed to connect to DB: abc12345.ap-south-1.snowflakecomputing.com:443. Role 'TEST_ROLE' specified in the connect string is not granted to this user. Contact your local system administrator, or attempt to login with another role, e.g. PUBLIC" ) - pipeline = Pipeline(snowflake_pipeline_config) - pipeline.run() - assert "permission-error" in pipeline.source.get_report().failures.keys() + with pytest.raises(PipelineInitError, match="Permissions error"): + pipeline = Pipeline(snowflake_pipeline_config) + pipeline.run() + pipeline.raise_from_status() @freeze_time(FROZEN_TIME) @@ -100,7 +98,9 @@ def test_snowflake_missing_warehouse_access_causes_pipeline_failure( ) pipeline = Pipeline(snowflake_pipeline_config) pipeline.run() - assert "permission-error" in pipeline.source.get_report().failures.keys() + assert "permission-error" in [ + failure.message for failure in pipeline.source.get_report().failures + ] @freeze_time(FROZEN_TIME) @@ -122,7 +122,9 @@ def test_snowflake_no_databases_with_access_causes_pipeline_failure( ) pipeline = Pipeline(snowflake_pipeline_config) pipeline.run() - assert "permission-error" in pipeline.source.get_report().failures.keys() + assert "permission-error" in [ + failure.message for failure in pipeline.source.get_report().failures + ] @freeze_time(FROZEN_TIME) @@ -144,13 +146,15 @@ def test_snowflake_no_tables_causes_pipeline_failure( ) sf_cursor.execute.side_effect = query_permission_response_override( no_tables_fn, - [SnowflakeQuery.show_views_for_schema("TEST_SCHEMA", "TEST_DB")], + [SnowflakeQuery.show_views_for_database("TEST_DB")], [], ) pipeline = Pipeline(snowflake_pipeline_config) pipeline.run() - assert "permission-error" in pipeline.source.get_report().failures.keys() + assert "permission-error" in [ + failure.message for failure in pipeline.source.get_report().failures + ] @freeze_time(FROZEN_TIME) @@ -168,20 +172,16 @@ def test_snowflake_list_columns_error_causes_pipeline_warning( sf_cursor.execute.side_effect = query_permission_error_override( default_query_results, [ - SnowflakeQuery.columns_for_table( - f"TABLE_{tbl_idx}", "TEST_SCHEMA", "TEST_DB" - ) - for tbl_idx in range(1, NUM_TABLES + 1) + SnowflakeQuery.columns_for_schema("TEST_SCHEMA", "TEST_DB"), ], "Database 'TEST_DB' does not exist or not authorized.", ) pipeline = Pipeline(snowflake_pipeline_config) pipeline.run() pipeline.raise_from_status() # pipeline should not fail - assert ( - "Failed to get columns for table" - in pipeline.source.get_report().warnings.keys() - ) + assert "Failed to get columns for table" in [ + warning.message for warning in pipeline.source.get_report().warnings + ] @freeze_time(FROZEN_TIME) @@ -204,10 +204,9 @@ def test_snowflake_list_primary_keys_error_causes_pipeline_warning( pipeline = Pipeline(snowflake_pipeline_config) pipeline.run() pipeline.raise_from_status() # pipeline should not fail - assert ( - "Failed to get primary key for table" - in pipeline.source.get_report().warnings.keys() - ) + assert "Failed to get primary key for table" in [ + warning.message for warning in pipeline.source.get_report().warnings + ] @freeze_time(FROZEN_TIME) @@ -228,7 +227,7 @@ def test_snowflake_missing_snowflake_lineage_permission_causes_pipeline_failure( snowflake_query.SnowflakeQuery.table_to_table_lineage_history_v2( start_time_millis=1654473600000, end_time_millis=1654586220000, - include_view_lineage=False, + include_view_lineage=True, include_column_lineage=True, ) ], @@ -236,9 +235,9 @@ def test_snowflake_missing_snowflake_lineage_permission_causes_pipeline_failure( ) pipeline = Pipeline(snowflake_pipeline_config) pipeline.run() - assert ( - "lineage-permission-error" in pipeline.source.get_report().failures.keys() - ) + assert "lineage-permission-error" in [ + failure.message for failure in pipeline.source.get_report().failures + ] @freeze_time(FROZEN_TIME) @@ -260,4 +259,6 @@ def test_snowflake_missing_snowflake_operations_permission_causes_pipeline_failu ) pipeline = Pipeline(snowflake_pipeline_config) pipeline.run() - assert "usage-permission-error" in pipeline.source.get_report().failures.keys() + assert "usage-permission-error" in [ + failure.message for failure in pipeline.source.get_report().failures + ] diff --git a/metadata-ingestion/tests/integration/superset/golden_test_ingest.json b/metadata-ingestion/tests/integration/superset/golden_test_ingest.json index 74312940f06e7..767b85a72b975 100644 --- a/metadata-ingestion/tests/integration/superset/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/superset/golden_test_ingest.json @@ -28,8 +28,8 @@ "datasets": [], "lastModified": { "created": { - "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_1" + "time": 0, + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1586847600000, @@ -44,7 +44,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-test" + "runId": "superset-test", + "lastRunId": "no-run-id-provided" } }, { @@ -74,8 +75,8 @@ "datasets": [], "lastModified": { "created": { - "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_2" + "time": 0, + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1586847600000, @@ -90,7 +91,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-test" + "runId": "superset-test", + "lastRunId": "no-run-id-provided" } }, { @@ -114,8 +116,8 @@ "description": "", "lastModified": { "created": { - "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_1" + "time": 0, + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1586847600000, @@ -136,7 +138,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-test" + "runId": "superset-test", + "lastRunId": "no-run-id-provided" } }, { @@ -160,8 +163,8 @@ "description": "", "lastModified": { "created": { - "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_1" + "time": 0, + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1586847600000, @@ -182,7 +185,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-test" + "runId": "superset-test", + "lastRunId": "no-run-id-provided" } }, { @@ -206,8 +210,8 @@ "description": "", "lastModified": { "created": { - "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_2" + "time": 0, + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1586847600000, @@ -228,7 +232,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-test" + "runId": "superset-test", + "lastRunId": "no-run-id-provided" } }, { @@ -252,8 +257,8 @@ "description": "", "lastModified": { "created": { - "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_2" + "time": 0, + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1586847600000, @@ -274,7 +279,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-test" + "runId": "superset-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json b/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json index cf38341085c1b..0f20799017979 100644 --- a/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json +++ b/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json @@ -28,8 +28,8 @@ "datasets": [], "lastModified": { "created": { - "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_1" + "time": 0, + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1586847600000, @@ -44,7 +44,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00" + "runId": "superset-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } }, { @@ -68,8 +70,8 @@ "description": "", "lastModified": { "created": { - "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_1" + "time": 0, + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1586847600000, @@ -90,7 +92,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00" + "runId": "superset-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } }, { @@ -114,8 +118,8 @@ "description": "", "lastModified": { "created": { - "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_1" + "time": 0, + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1586847600000, @@ -136,7 +140,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00" + "runId": "superset-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } }, { @@ -160,8 +166,8 @@ "description": "", "lastModified": { "created": { - "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_2" + "time": 0, + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1586847600000, @@ -182,7 +188,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00" + "runId": "superset-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } }, { @@ -206,8 +214,8 @@ "description": "", "lastModified": { "created": { - "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_2" + "time": 0, + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1586847600000, @@ -228,7 +236,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00" + "runId": "superset-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } }, { @@ -243,7 +253,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00" + "runId": "superset-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json index d8c27057872c8..5de4fe5f647d9 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json @@ -16,7 +16,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -32,7 +33,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -48,7 +50,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -66,7 +69,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -82,7 +86,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -102,7 +107,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -118,7 +124,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -134,7 +141,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -152,7 +160,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -168,7 +177,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -188,7 +198,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -204,7 +215,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -220,7 +232,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -238,7 +251,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -254,7 +268,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -276,7 +291,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -292,7 +308,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -308,7 +325,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -326,7 +344,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -342,6 +361,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -351,7 +371,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -367,7 +388,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -388,7 +410,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -410,7 +433,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -426,7 +450,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -442,7 +467,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -460,7 +486,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -476,6 +503,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -485,7 +513,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -505,7 +534,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -521,7 +551,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -542,7 +573,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -564,7 +596,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -580,7 +613,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -596,7 +630,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -614,7 +649,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -630,6 +666,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -639,7 +676,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -655,7 +693,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -676,7 +715,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -698,7 +738,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -714,7 +755,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -730,7 +772,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -748,7 +791,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -764,6 +808,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -773,7 +818,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -789,7 +835,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -810,7 +857,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -831,7 +879,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -849,7 +898,7 @@ "customProperties": { "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783" }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline - Sent", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent", "title": "Timeline - Sent", "description": "", "lastModified": { @@ -884,6 +933,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -896,7 +946,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -912,7 +963,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -1085,7 +1137,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -1110,7 +1163,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -1126,7 +1180,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign List", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List", "title": "Campaign List", "description": "", "lastModified": { @@ -1161,6 +1215,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1173,7 +1228,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -1189,7 +1245,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -1648,7 +1705,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -1673,7 +1731,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -1724,6 +1783,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1736,7 +1796,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -1752,7 +1813,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -2263,7 +2325,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -2288,7 +2351,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -2304,7 +2368,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile - Sent by Campaign", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign", "title": "Mobile - Sent by Campaign", "description": "", "lastModified": { @@ -2339,6 +2403,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2351,7 +2416,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -2367,7 +2433,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -2774,7 +2841,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -2799,7 +2867,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -2850,6 +2919,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2862,7 +2932,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -2878,7 +2949,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -2921,7 +2993,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -2946,7 +3019,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -2997,6 +3071,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3009,7 +3084,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -3025,7 +3101,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -3236,7 +3313,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -3261,7 +3339,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -3312,6 +3391,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3333,7 +3413,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -3349,7 +3430,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -3435,7 +3517,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -3460,7 +3543,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -3476,7 +3560,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Requests", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests", "title": "Opened Requests", "description": "", "lastModified": { @@ -3514,6 +3598,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3526,7 +3611,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -3542,7 +3628,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -3938,7 +4025,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -3963,7 +4051,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -3979,7 +4068,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top 10 Items by Requests and YoY Change", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change", "title": "Top 10 Items by Requests and YoY Change", "description": "", "lastModified": { @@ -4017,6 +4106,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4029,7 +4119,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -4045,7 +4136,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -4409,7 +4501,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -4434,7 +4527,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -4450,7 +4544,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems", "title": "Opened Problems", "description": "", "lastModified": { @@ -4488,6 +4582,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4500,7 +4595,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -4516,7 +4612,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -4912,7 +5009,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -4937,7 +5035,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -4988,6 +5087,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5000,7 +5100,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -5016,7 +5117,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -5296,7 +5398,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -5321,7 +5424,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -5337,7 +5441,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High and Critical Priority Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems", "title": "High and Critical Priority Problems", "description": "", "lastModified": { @@ -5375,6 +5479,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5387,7 +5492,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -5403,7 +5509,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -5660,7 +5767,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -5685,7 +5793,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -5701,7 +5810,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total Incidents by Category and YoY Change", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change", "title": "Total Incidents by Category and YoY Change", "description": "", "lastModified": { @@ -5736,6 +5845,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5748,7 +5858,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -5764,7 +5875,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -6047,7 +6159,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -6072,7 +6185,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -6088,7 +6202,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known Errors", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors", "title": "Known Errors", "description": "", "lastModified": { @@ -6126,6 +6240,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6138,7 +6253,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -6154,7 +6270,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -6489,7 +6606,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -6514,7 +6632,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -6530,7 +6649,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Requests", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests", "title": "Overdue Requests", "description": "", "lastModified": { @@ -6568,6 +6687,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6580,7 +6700,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -6596,7 +6717,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -6905,7 +7027,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -6930,7 +7053,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -6946,7 +7070,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG Time to Solve an Incident", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident", "title": "AVG Time to Solve an Incident", "description": "", "lastModified": { @@ -6981,6 +7105,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6993,7 +7118,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -7009,7 +7135,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -7237,7 +7364,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -7262,7 +7390,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -7278,7 +7407,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made SLA?", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F", "title": "Made SLA?", "description": "", "lastModified": { @@ -7316,6 +7445,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7328,7 +7458,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -7344,7 +7475,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -7601,7 +7733,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -7626,7 +7759,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -7677,6 +7811,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7689,7 +7824,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -7705,7 +7841,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -7855,7 +7992,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -7880,7 +8018,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -7896,7 +8035,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems", "title": "Overdue Problems", "description": "", "lastModified": { @@ -7934,6 +8073,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7946,7 +8086,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -7962,7 +8103,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -8297,7 +8439,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -8322,7 +8465,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -8376,6 +8520,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8388,7 +8533,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -8404,7 +8550,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -8635,7 +8782,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -8660,7 +8808,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -8714,6 +8863,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8726,7 +8876,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -8742,7 +8893,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -8999,7 +9151,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -9024,7 +9177,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -9040,7 +9194,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age of Active Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems", "title": "Age of Active Problems", "description": "", "lastModified": { @@ -9078,6 +9232,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9090,7 +9245,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -9106,7 +9262,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -9389,7 +9546,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -9414,7 +9572,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -9430,7 +9589,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Incidents", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents", "title": "Opened Incidents", "description": "", "lastModified": { @@ -9465,6 +9624,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9477,7 +9637,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -9493,7 +9654,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -9831,7 +9993,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -9856,7 +10019,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -9907,6 +10071,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9919,7 +10084,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -9935,7 +10101,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10021,7 +10188,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10046,7 +10214,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10067,7 +10236,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10122,6 +10292,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10134,7 +10305,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10150,7 +10322,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10175,7 +10348,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10225,6 +10399,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10237,7 +10412,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10253,7 +10429,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10278,7 +10455,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10326,6 +10504,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10338,7 +10517,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10354,7 +10534,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10379,7 +10560,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10441,6 +10623,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10453,7 +10636,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10469,7 +10653,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10494,7 +10679,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10838,7 +11024,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -10866,6 +11053,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -12629,7 +12817,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -12647,7 +12836,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -12663,7 +12853,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -12688,7 +12879,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -12789,7 +12981,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -12817,6 +13010,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -13032,7 +13226,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -13050,7 +13245,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -13066,7 +13262,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -13091,7 +13288,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -13124,7 +13322,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -13152,6 +13351,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -13539,7 +13739,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -13557,7 +13758,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -13573,7 +13775,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -13598,7 +13801,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -14033,7 +14237,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -14061,6 +14266,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -14783,7 +14989,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -14801,7 +15008,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -14817,7 +15025,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -14842,7 +15051,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -14988,7 +15198,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -15016,6 +15227,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -21606,7 +21818,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -21624,7 +21837,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -21640,7 +21854,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -21665,7 +21880,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -21854,7 +22070,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -21882,6 +22099,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -25758,7 +25976,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -25776,7 +25995,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -25792,7 +26012,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -25817,7 +26038,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -25968,7 +26190,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -25996,6 +26219,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31134,7 +31358,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -31152,7 +31377,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -31168,7 +31394,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -31193,7 +31420,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -31316,7 +31544,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -31344,6 +31573,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31546,7 +31776,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -31564,7 +31795,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -31580,7 +31812,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -31605,7 +31838,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -31654,7 +31888,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -31682,6 +31917,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31774,7 +32010,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -31792,7 +32029,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -31808,7 +32046,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -31833,7 +32072,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -31937,7 +32177,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -31974,6 +32215,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -32222,7 +32464,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -32240,7 +32483,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -32256,7 +32500,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -32277,7 +32522,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -32411,7 +32657,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -32439,6 +32686,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -33291,7 +33539,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33309,7 +33558,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33325,7 +33575,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33346,7 +33597,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33362,7 +33614,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33395,7 +33648,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33530,7 +33784,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33549,7 +33804,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33574,7 +33830,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33590,7 +33847,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33631,7 +33889,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33778,7 +34037,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33797,7 +34057,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33818,7 +34079,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33871,7 +34133,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33958,7 +34221,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -33977,7 +34241,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -34000,7 +34265,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -34164,7 +34430,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -34190,7 +34457,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -34426,7 +34694,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -34452,7 +34721,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -34664,7 +34934,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -34690,7 +34961,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -34854,7 +35126,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -34880,7 +35153,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -35032,7 +35306,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -35058,7 +35333,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -35198,7 +35474,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -35224,7 +35501,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -35316,7 +35594,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -35342,7 +35621,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -35410,7 +35690,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -35433,7 +35714,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -35501,7 +35783,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -35524,7 +35807,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -35820,7 +36104,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -35843,7 +36128,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -36633,7 +36919,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -36659,7 +36946,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -37543,7 +37831,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -37569,7 +37858,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -38525,7 +38815,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -38551,7 +38842,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -39375,7 +39667,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -39401,7 +39694,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -39709,7 +40003,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -39735,7 +40030,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -40583,7 +40879,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -40609,7 +40906,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -41601,7 +41899,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -41627,7 +41926,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42487,7 +42787,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42513,7 +42814,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42535,7 +42837,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42558,7 +42861,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42580,7 +42884,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42603,7 +42908,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42624,7 +42930,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42647,7 +42954,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42663,7 +42971,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42679,7 +42988,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42695,7 +43005,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42711,7 +43022,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42727,7 +43039,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42743,7 +43056,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42759,7 +43073,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42775,7 +43090,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42791,7 +43107,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42807,7 +43124,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42823,7 +43141,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42839,7 +43158,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42855,7 +43175,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42871,7 +43192,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42887,7 +43209,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42903,7 +43226,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42919,7 +43243,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42935,7 +43260,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42951,7 +43277,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42967,7 +43294,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42983,7 +43311,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -42999,7 +43328,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43015,7 +43345,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43031,7 +43362,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43047,7 +43379,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43063,7 +43396,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43079,7 +43413,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43095,7 +43430,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43111,7 +43447,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43127,7 +43464,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43143,7 +43481,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43159,7 +43498,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43175,7 +43515,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43191,7 +43532,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43207,7 +43549,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43223,7 +43566,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43239,7 +43583,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43255,7 +43600,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43271,7 +43617,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43287,7 +43634,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43303,7 +43651,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43319,7 +43668,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43335,7 +43685,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43351,7 +43702,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43367,7 +43719,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43383,7 +43736,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43399,7 +43753,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43415,7 +43770,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43431,7 +43787,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43447,7 +43804,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43463,7 +43821,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43479,7 +43838,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43495,7 +43855,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43511,7 +43872,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43527,7 +43889,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43543,7 +43906,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43559,7 +43923,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43575,7 +43940,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43591,7 +43957,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43607,7 +43974,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43623,7 +43991,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43639,7 +44008,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43655,7 +44025,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43671,7 +44042,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43687,7 +44059,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43703,7 +44076,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43719,7 +44093,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43735,7 +44110,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43751,7 +44127,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43767,7 +44144,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43783,7 +44161,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43799,7 +44178,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43815,7 +44195,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43831,7 +44212,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43847,7 +44229,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43863,7 +44246,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43879,7 +44263,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43895,7 +44280,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43911,7 +44297,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } }, { @@ -43927,1703 +44314,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", - "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "SubProject1" - }, - { - "id": "AbcJoinWorkbook" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Dvdrental Workbook" - }, - { - "id": "actor+ (dvdrental)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Dvdrental Workbook" - }, - { - "id": "actor+ (dvdrental)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Incidents" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Incidents" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Customer Payment Query" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Customer Payment Query" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "test publish datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_cll_ingest" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json index 250c43b7fc2da..3fd5ed1ac9dee 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json @@ -16,7 +16,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -32,7 +33,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -48,7 +50,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -66,7 +69,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -82,7 +86,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -102,7 +107,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -118,7 +124,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -134,7 +141,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -152,7 +160,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -168,7 +177,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -188,7 +198,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -204,7 +215,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -220,7 +232,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -238,7 +251,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -254,7 +268,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -274,7 +289,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -290,7 +306,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -306,7 +323,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -324,7 +342,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -340,7 +359,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -361,7 +381,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -383,7 +404,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -399,7 +421,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -415,7 +438,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -433,7 +457,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -449,6 +474,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -458,7 +484,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -474,7 +501,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -495,7 +523,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -517,7 +546,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -533,7 +563,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -549,7 +580,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -567,7 +599,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -583,6 +616,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -592,7 +626,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -612,7 +647,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -628,7 +664,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -649,7 +686,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -671,7 +709,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -687,7 +726,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -703,7 +743,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -721,7 +762,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -737,6 +779,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -746,7 +789,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -762,7 +806,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -783,7 +828,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -805,7 +851,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -821,7 +868,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -837,7 +885,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -855,7 +904,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -871,6 +921,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -880,7 +931,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -896,7 +948,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -917,7 +970,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -939,7 +993,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -955,7 +1010,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -971,7 +1027,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -989,7 +1046,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -1005,6 +1063,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1014,7 +1073,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -1030,7 +1090,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -1055,7 +1116,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -1076,7 +1138,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -1094,7 +1157,7 @@ "customProperties": { "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783" }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline - Sent", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent", "title": "Timeline - Sent", "description": "", "lastModified": { @@ -1129,6 +1192,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1141,7 +1205,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -1157,7 +1222,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -1330,7 +1396,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -1355,7 +1422,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -1371,7 +1439,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign List", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List", "title": "Campaign List", "description": "", "lastModified": { @@ -1406,6 +1474,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1418,7 +1487,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -1434,7 +1504,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -1893,7 +1964,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -1918,7 +1990,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -1969,6 +2042,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1981,7 +2055,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -1997,7 +2072,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -2508,7 +2584,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -2533,7 +2610,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -2549,7 +2627,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile - Sent by Campaign", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign", "title": "Mobile - Sent by Campaign", "description": "", "lastModified": { @@ -2584,6 +2662,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2596,7 +2675,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -2612,7 +2692,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3019,7 +3100,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3044,7 +3126,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3095,6 +3178,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3107,7 +3191,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3123,7 +3208,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3166,7 +3252,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3191,7 +3278,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3242,6 +3330,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3254,7 +3343,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3270,7 +3360,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3481,7 +3572,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3506,7 +3598,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3557,6 +3650,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3578,7 +3672,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3594,7 +3689,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3680,7 +3776,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3705,7 +3802,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3721,7 +3819,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Requests", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests", "title": "Opened Requests", "description": "", "lastModified": { @@ -3759,6 +3857,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3771,7 +3870,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -3787,7 +3887,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -4183,7 +4284,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -4208,7 +4310,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -4224,7 +4327,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top 10 Items by Requests and YoY Change", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change", "title": "Top 10 Items by Requests and YoY Change", "description": "", "lastModified": { @@ -4262,6 +4365,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4274,7 +4378,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -4290,7 +4395,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -4654,7 +4760,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -4679,7 +4786,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -4695,7 +4803,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems", "title": "Opened Problems", "description": "", "lastModified": { @@ -4733,6 +4841,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4745,7 +4854,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -4761,7 +4871,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -5157,7 +5268,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -5182,7 +5294,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -5233,6 +5346,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5245,7 +5359,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -5261,7 +5376,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -5541,7 +5657,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -5566,7 +5683,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -5582,7 +5700,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High and Critical Priority Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems", "title": "High and Critical Priority Problems", "description": "", "lastModified": { @@ -5620,6 +5738,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5632,7 +5751,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -5648,7 +5768,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -5905,7 +6026,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -5930,7 +6052,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -5946,7 +6069,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total Incidents by Category and YoY Change", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change", "title": "Total Incidents by Category and YoY Change", "description": "", "lastModified": { @@ -5981,6 +6104,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5993,7 +6117,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -6009,7 +6134,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -6292,7 +6418,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -6317,7 +6444,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -6333,7 +6461,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known Errors", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors", "title": "Known Errors", "description": "", "lastModified": { @@ -6371,6 +6499,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6383,7 +6512,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -6399,7 +6529,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -6734,7 +6865,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -6759,7 +6891,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -6775,7 +6908,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Requests", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests", "title": "Overdue Requests", "description": "", "lastModified": { @@ -6813,6 +6946,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6825,7 +6959,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -6841,7 +6976,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -7150,7 +7286,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -7175,7 +7312,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -7191,7 +7329,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG Time to Solve an Incident", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident", "title": "AVG Time to Solve an Incident", "description": "", "lastModified": { @@ -7226,6 +7364,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7238,7 +7377,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -7254,7 +7394,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -7482,7 +7623,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -7507,7 +7649,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -7523,7 +7666,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made SLA?", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F", "title": "Made SLA?", "description": "", "lastModified": { @@ -7561,6 +7704,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7573,7 +7717,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -7589,7 +7734,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -7846,7 +7992,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -7871,7 +8018,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -7922,6 +8070,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7934,7 +8083,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -7950,7 +8100,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -8100,7 +8251,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -8125,7 +8277,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -8141,7 +8294,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems", "title": "Overdue Problems", "description": "", "lastModified": { @@ -8179,6 +8332,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8191,7 +8345,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -8207,7 +8362,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -8542,7 +8698,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -8567,7 +8724,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -8621,6 +8779,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8633,7 +8792,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -8649,7 +8809,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -8880,7 +9041,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -8905,7 +9067,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -8959,6 +9122,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8971,7 +9135,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -8987,7 +9152,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -9244,7 +9410,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -9269,7 +9436,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -9285,7 +9453,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age of Active Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems", "title": "Age of Active Problems", "description": "", "lastModified": { @@ -9323,6 +9491,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9335,7 +9504,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -9351,7 +9521,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -9634,7 +9805,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -9659,7 +9831,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -9675,7 +9848,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Incidents", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents", "title": "Opened Incidents", "description": "", "lastModified": { @@ -9710,6 +9883,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9722,7 +9896,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -9738,7 +9913,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10076,7 +10252,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10101,7 +10278,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10152,6 +10330,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10164,7 +10343,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10180,7 +10360,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10266,7 +10447,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10291,7 +10473,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10312,7 +10495,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10367,6 +10551,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10379,7 +10564,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10395,7 +10581,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10420,7 +10607,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10470,6 +10658,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10482,7 +10671,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10498,7 +10688,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10523,7 +10714,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10571,6 +10763,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10583,7 +10776,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10599,7 +10793,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10624,7 +10819,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10686,6 +10882,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10698,7 +10895,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10714,7 +10912,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -10739,7 +10938,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -11083,7 +11283,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -11111,6 +11312,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -12874,7 +13076,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -12892,7 +13095,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -12908,7 +13112,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -12933,7 +13138,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -13034,7 +13240,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -13062,6 +13269,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -13277,7 +13485,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -13295,7 +13504,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -13311,7 +13521,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -13336,7 +13547,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -13369,7 +13581,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -13397,6 +13610,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -13784,7 +13998,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -13802,7 +14017,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -13818,7 +14034,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -13843,7 +14060,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -14278,7 +14496,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -14306,6 +14525,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -15028,7 +15248,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -15046,7 +15267,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -15062,7 +15284,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -15087,7 +15310,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -15233,7 +15457,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -15261,6 +15486,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -21851,7 +22077,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -21869,7 +22096,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -21885,7 +22113,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -21910,7 +22139,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -22099,7 +22329,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -22127,6 +22358,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -26003,7 +26235,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -26021,7 +26254,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -26037,7 +26271,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -26062,7 +26297,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -26213,7 +26449,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -26241,6 +26478,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31379,7 +31617,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -31397,7 +31636,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -31413,7 +31653,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -31438,7 +31679,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -31561,7 +31803,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -31589,6 +31832,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31791,7 +32035,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -31809,7 +32054,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -31825,7 +32071,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -31850,7 +32097,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -31899,7 +32147,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -31927,6 +32176,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -32019,7 +32269,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -32037,7 +32288,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -32053,7 +32305,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -32078,7 +32331,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -32182,7 +32436,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -32219,6 +32474,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -32467,7 +32723,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -32485,7 +32742,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -32501,7 +32759,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -32522,7 +32781,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -32656,7 +32916,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -32684,6 +32945,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -33536,7 +33798,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -33554,7 +33817,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -33570,7 +33834,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -33591,7 +33856,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -33607,7 +33873,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -33640,7 +33907,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -33775,7 +34043,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -33794,7 +34063,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -33819,7 +34089,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -33835,7 +34106,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -33876,7 +34148,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -34023,7 +34296,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -34042,7 +34316,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -34063,7 +34338,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -34150,7 +34426,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -34169,7 +34446,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -34192,7 +34470,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -34356,7 +34635,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -34382,7 +34662,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -34618,7 +34899,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -34644,7 +34926,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -34856,7 +35139,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -34882,7 +35166,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -35046,7 +35331,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -35072,7 +35358,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -35224,7 +35511,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -35250,7 +35538,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -35390,7 +35679,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -35416,7 +35706,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -35508,7 +35799,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -35534,7 +35826,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -35602,7 +35895,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -35625,7 +35919,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -35693,7 +35988,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -35716,7 +36012,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -36012,7 +36309,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -36035,7 +36333,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -36825,7 +37124,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -36851,7 +37151,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -37735,7 +38036,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -37761,7 +38063,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -38717,7 +39020,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -38743,7 +39047,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -39567,7 +39872,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -39593,7 +39899,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -39901,7 +40208,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -39927,7 +40235,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -40775,7 +41084,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -40801,7 +41111,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -41793,7 +42104,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -41819,7 +42131,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42679,7 +42992,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42705,7 +43019,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42727,7 +43042,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42750,7 +43066,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42772,7 +43089,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42795,7 +43113,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42816,7 +43135,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42839,7 +43159,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42855,7 +43176,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42871,7 +43193,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42887,7 +43210,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42903,7 +43227,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42919,7 +43244,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42935,7 +43261,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42951,7 +43278,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42967,7 +43295,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42983,7 +43312,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -42999,7 +43329,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43015,7 +43346,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43031,7 +43363,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43047,7 +43380,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43063,7 +43397,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43079,7 +43414,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43095,7 +43431,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43111,7 +43448,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43127,7 +43465,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43143,7 +43482,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43159,7 +43499,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43175,7 +43516,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43191,7 +43533,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43207,7 +43550,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43223,7 +43567,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43239,7 +43584,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43255,7 +43601,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43271,7 +43618,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43287,7 +43635,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43303,7 +43652,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43319,7 +43669,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43335,7 +43686,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43351,7 +43703,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43367,7 +43720,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43383,7 +43737,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43399,7 +43754,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43415,7 +43771,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43431,7 +43788,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43447,7 +43805,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43463,7 +43822,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43479,7 +43839,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43495,7 +43856,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43511,7 +43873,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43527,7 +43890,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43543,7 +43907,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43559,7 +43924,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43575,7 +43941,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43591,7 +43958,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43607,7 +43975,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43623,7 +43992,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43639,7 +44009,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43655,7 +44026,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43671,7 +44043,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43687,7 +44060,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43703,7 +44077,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43719,7 +44094,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43735,7 +44111,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43751,7 +44128,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43767,7 +44145,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43783,7 +44162,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43799,7 +44179,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43815,7 +44196,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43831,7 +44213,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43847,7 +44230,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43863,7 +44247,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43879,7 +44264,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43895,7 +44281,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43911,7 +44298,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43927,7 +44315,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43943,7 +44332,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43959,7 +44349,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43975,7 +44366,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -43991,7 +44383,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -44007,7 +44400,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -44023,7 +44417,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -44039,7 +44434,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -44055,7 +44451,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -44071,7 +44468,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } }, { @@ -44087,1749 +44485,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", - "urn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", - "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "SubProject1" - }, - { - "id": "AbcJoinWorkbook" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Dvdrental Workbook" - }, - { - "id": "actor+ (dvdrental)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Dvdrental Workbook" - }, - { - "id": "actor+ (dvdrental)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Incidents" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Incidents" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Customer Payment Query" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Customer Payment Query" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "test publish datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "tableau-test-pipeline" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json index c5417a8d212bc..29e1b77869aa5 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json @@ -16,7 +16,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -32,7 +33,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -48,7 +50,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -66,7 +69,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -82,7 +86,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -102,7 +107,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -118,7 +124,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -134,7 +141,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -152,7 +160,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -168,7 +177,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -188,7 +198,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -204,7 +215,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -220,7 +232,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -238,7 +251,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -254,7 +268,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -276,7 +291,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -292,7 +308,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -308,7 +325,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -326,7 +344,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -342,6 +361,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -351,7 +371,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -367,7 +388,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -388,7 +410,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -410,7 +433,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -426,7 +450,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -442,7 +467,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -460,7 +486,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -476,6 +503,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -485,7 +513,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -505,7 +534,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -521,7 +551,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -542,7 +573,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -564,7 +596,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -580,7 +613,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -596,7 +630,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -614,7 +649,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -630,6 +666,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -639,7 +676,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -655,7 +693,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -676,7 +715,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -698,7 +738,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -714,7 +755,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -730,7 +772,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -748,7 +791,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -764,6 +808,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -773,7 +818,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -789,7 +835,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -810,7 +857,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -831,7 +879,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -849,7 +898,7 @@ "customProperties": { "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783" }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline - Sent", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent", "title": "Timeline - Sent", "description": "", "lastModified": { @@ -884,6 +933,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -896,7 +946,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -912,7 +963,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -1085,7 +1137,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -1110,7 +1163,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -1126,7 +1180,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign List", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List", "title": "Campaign List", "description": "", "lastModified": { @@ -1161,6 +1215,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1173,7 +1228,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -1189,7 +1245,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -1648,7 +1705,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -1673,7 +1731,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -1724,6 +1783,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1736,7 +1796,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -1752,7 +1813,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -2263,7 +2325,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -2288,7 +2351,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -2304,7 +2368,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile - Sent by Campaign", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign", "title": "Mobile - Sent by Campaign", "description": "", "lastModified": { @@ -2339,6 +2403,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2351,7 +2416,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -2367,7 +2433,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -2774,7 +2841,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -2799,7 +2867,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -2850,6 +2919,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2862,7 +2932,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -2878,7 +2949,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -2921,7 +2993,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -2946,7 +3019,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -2997,6 +3071,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3009,7 +3084,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -3025,7 +3101,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -3236,7 +3313,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -3261,7 +3339,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -3312,6 +3391,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3333,7 +3413,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -3349,7 +3430,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -3435,7 +3517,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -3460,7 +3543,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -3476,7 +3560,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Requests", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests", "title": "Opened Requests", "description": "", "lastModified": { @@ -3514,6 +3598,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3526,7 +3611,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -3542,7 +3628,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -3938,7 +4025,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -3963,7 +4051,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -3979,7 +4068,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top 10 Items by Requests and YoY Change", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change", "title": "Top 10 Items by Requests and YoY Change", "description": "", "lastModified": { @@ -4017,6 +4106,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4029,7 +4119,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -4045,7 +4136,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -4409,7 +4501,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -4434,7 +4527,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -4450,7 +4544,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems", "title": "Opened Problems", "description": "", "lastModified": { @@ -4488,6 +4582,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4500,7 +4595,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -4516,7 +4612,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -4912,7 +5009,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -4937,7 +5035,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -4988,6 +5087,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5000,7 +5100,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -5016,7 +5117,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -5296,7 +5398,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -5321,7 +5424,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -5337,7 +5441,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High and Critical Priority Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems", "title": "High and Critical Priority Problems", "description": "", "lastModified": { @@ -5375,6 +5479,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5387,7 +5492,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -5403,7 +5509,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -5660,7 +5767,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -5685,7 +5793,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -5701,7 +5810,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total Incidents by Category and YoY Change", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change", "title": "Total Incidents by Category and YoY Change", "description": "", "lastModified": { @@ -5736,6 +5845,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5748,7 +5858,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -5764,7 +5875,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -6047,7 +6159,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -6072,7 +6185,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -6088,7 +6202,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known Errors", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors", "title": "Known Errors", "description": "", "lastModified": { @@ -6126,6 +6240,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6138,7 +6253,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -6154,7 +6270,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -6489,7 +6606,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -6514,7 +6632,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -6530,7 +6649,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Requests", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests", "title": "Overdue Requests", "description": "", "lastModified": { @@ -6568,6 +6687,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6580,7 +6700,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -6596,7 +6717,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -6905,7 +7027,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -6930,7 +7053,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -6946,7 +7070,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG Time to Solve an Incident", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident", "title": "AVG Time to Solve an Incident", "description": "", "lastModified": { @@ -6981,6 +7105,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6993,7 +7118,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -7009,7 +7135,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -7237,7 +7364,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -7262,7 +7390,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -7278,7 +7407,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made SLA?", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F", "title": "Made SLA?", "description": "", "lastModified": { @@ -7316,6 +7445,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7328,7 +7458,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -7344,7 +7475,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -7601,7 +7733,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -7626,7 +7759,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -7677,6 +7811,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7689,7 +7824,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -7705,7 +7841,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -7855,7 +7992,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -7880,7 +8018,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -7896,7 +8035,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems", "title": "Overdue Problems", "description": "", "lastModified": { @@ -7934,6 +8073,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7946,7 +8086,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -7962,7 +8103,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -8297,7 +8439,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -8322,7 +8465,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -8376,6 +8520,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8388,7 +8533,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -8404,7 +8550,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -8635,7 +8782,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -8660,7 +8808,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -8714,6 +8863,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8726,7 +8876,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -8742,7 +8893,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -8999,7 +9151,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -9024,7 +9177,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -9040,7 +9194,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age of Active Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems", "title": "Age of Active Problems", "description": "", "lastModified": { @@ -9078,6 +9232,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9090,7 +9245,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -9106,7 +9262,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -9389,7 +9546,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -9414,7 +9572,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -9430,7 +9589,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Incidents", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents", "title": "Opened Incidents", "description": "", "lastModified": { @@ -9465,6 +9624,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9477,7 +9637,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -9493,7 +9654,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -9831,7 +9993,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -9856,7 +10019,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -9907,6 +10071,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9919,7 +10084,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -9935,7 +10101,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10021,7 +10188,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10046,7 +10214,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10067,7 +10236,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10122,6 +10292,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10134,7 +10305,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10150,7 +10322,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10175,7 +10348,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10225,6 +10399,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10237,7 +10412,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10253,7 +10429,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10278,7 +10455,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10326,6 +10504,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10338,7 +10517,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10354,7 +10534,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10379,7 +10560,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10441,6 +10623,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10453,7 +10636,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10469,7 +10653,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10494,7 +10679,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10838,7 +11024,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -10866,6 +11053,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -12629,7 +12817,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -12647,7 +12836,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -12663,7 +12853,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -12688,7 +12879,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -12789,7 +12981,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -12817,6 +13010,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -13032,7 +13226,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -13050,7 +13245,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -13066,7 +13262,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -13091,7 +13288,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -13124,7 +13322,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -13152,6 +13351,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -13539,7 +13739,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -13557,7 +13758,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -13573,7 +13775,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -13598,7 +13801,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -14033,7 +14237,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -14061,6 +14266,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -14783,7 +14989,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -14801,7 +15008,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -14817,7 +15025,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -14842,7 +15051,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -14988,7 +15198,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -15016,6 +15227,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -21606,7 +21818,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -21624,7 +21837,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -21640,7 +21854,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -21665,7 +21880,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -21854,7 +22070,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -21882,6 +22099,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -25758,7 +25976,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -25776,7 +25995,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -25792,7 +26012,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -25817,7 +26038,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -25968,7 +26190,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -25996,6 +26219,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31134,7 +31358,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -31152,7 +31377,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -31168,7 +31394,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -31193,7 +31420,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -31316,7 +31544,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -31344,6 +31573,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31546,7 +31776,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -31564,7 +31795,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -31580,7 +31812,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -31605,7 +31838,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -31654,7 +31888,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -31682,6 +31917,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31774,7 +32010,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -31792,7 +32029,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -31808,7 +32046,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -31833,7 +32072,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -31937,7 +32177,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -31974,6 +32215,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -32222,7 +32464,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -32240,7 +32483,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -32256,7 +32500,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -32277,7 +32522,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -32411,7 +32657,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -32439,6 +32686,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -33291,7 +33539,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33309,7 +33558,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33325,7 +33575,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33346,7 +33597,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33362,7 +33614,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33395,7 +33648,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33530,7 +33784,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33549,7 +33804,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33574,7 +33830,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33590,7 +33847,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33631,7 +33889,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33778,7 +34037,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33797,7 +34057,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33818,7 +34079,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33905,7 +34167,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33924,7 +34187,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -33947,7 +34211,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -34111,7 +34376,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -34137,7 +34403,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -34373,7 +34640,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -34399,7 +34667,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -34611,7 +34880,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -34637,7 +34907,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -34801,7 +35072,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -34827,7 +35099,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -34979,7 +35252,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -35005,7 +35279,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -35145,7 +35420,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -35171,7 +35447,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -35263,7 +35540,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -35289,7 +35567,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -35357,7 +35636,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -35380,7 +35660,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -35448,7 +35729,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -35471,7 +35753,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -35767,7 +36050,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -35790,7 +36074,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -36580,7 +36865,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -36606,7 +36892,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -37490,7 +37777,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -37516,7 +37804,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -38472,7 +38761,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -38498,7 +38788,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -39322,7 +39613,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -39348,7 +39640,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -39656,7 +39949,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -39682,7 +39976,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -40530,7 +40825,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -40556,7 +40852,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -41548,7 +41845,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -41574,7 +41872,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42434,7 +42733,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42460,7 +42760,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42482,7 +42783,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42505,7 +42807,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42527,7 +42830,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42550,7 +42854,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42571,7 +42876,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42594,7 +42900,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42610,7 +42917,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42626,7 +42934,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42642,7 +42951,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42658,7 +42968,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42674,7 +42985,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42690,7 +43002,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42706,7 +43019,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42722,7 +43036,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42738,7 +43053,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42754,7 +43070,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42770,7 +43087,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42786,7 +43104,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42802,7 +43121,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42818,7 +43138,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42834,7 +43155,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42850,7 +43172,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42866,7 +43189,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42882,7 +43206,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42898,7 +43223,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42914,7 +43240,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42930,7 +43257,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42946,7 +43274,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42962,7 +43291,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42978,7 +43308,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -42994,7 +43325,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43010,7 +43342,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43026,7 +43359,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43042,7 +43376,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43058,7 +43393,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43074,7 +43410,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43090,7 +43427,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43106,7 +43444,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43122,7 +43461,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43138,7 +43478,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43154,7 +43495,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43170,7 +43512,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43186,7 +43529,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43202,7 +43546,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43218,7 +43563,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43234,7 +43580,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43250,7 +43597,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43266,7 +43614,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43282,7 +43631,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43298,7 +43648,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43314,7 +43665,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43330,7 +43682,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43346,7 +43699,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43362,7 +43716,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43378,7 +43733,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43394,7 +43750,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43410,7 +43767,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43426,7 +43784,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43442,7 +43801,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43458,7 +43818,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43474,7 +43835,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43490,7 +43852,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43506,7 +43869,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43522,7 +43886,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43538,7 +43903,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43554,7 +43920,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43570,7 +43937,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43586,7 +43954,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43602,7 +43971,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43618,7 +43988,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43634,7 +44005,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43650,7 +44022,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43666,7 +44039,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43682,7 +44056,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43698,7 +44073,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43714,7 +44090,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43730,7 +44107,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43746,7 +44124,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43762,7 +44141,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43778,7 +44158,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43794,7 +44175,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43810,7 +44192,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43826,7 +44209,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } }, { @@ -43842,1703 +44226,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", - "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "SubProject1" - }, - { - "id": "AbcJoinWorkbook" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Dvdrental Workbook" - }, - { - "id": "actor+ (dvdrental)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Dvdrental Workbook" - }, - { - "id": "actor+ (dvdrental)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Incidents" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Incidents" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Customer Payment Query" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Customer Payment Query" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "test publish datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_multiple_sites_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_multiple_sites_mces_golden.json new file mode 100644 index 0000000000000..12c665235e6ac --- /dev/null +++ b/metadata-ingestion/tests/integration/tableau/tableau_multiple_sites_mces_golden.json @@ -0,0 +1,85917 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "site_id": "190a6a5c-63ed-4de1-8045-site1" + }, + "name": "Acryl" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Site" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "190a6a5c-63ed-4de1-8045-faeae5df5b01" + }, + "name": "default" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "c30aafe5-44f4-4f28-80d3-d181010a263c" + }, + "name": "Project 2" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "910733aa-2e95-4ac3-a2e8-71570751099d" + }, + "name": "Samples" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "1f15d897-7f0c-7c59-037a-afa6a9b7c9a9" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15995", + "name": "Email Performance by Campaign", + "description": "Description for Email Performance by Campaign" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "661fabd0-bed6-8610-e066-0694a81a6cea" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15619", + "name": "Dvdrental Workbook", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:TagSheet3" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "6ffa5a7f-d852-78f1-6c6d-20ac23610ebf" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15605", + "name": "Executive Dashboard", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "bd040833-8f66-22c0-1b51-bd4ccf5eef7c" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/17904", + "name": "Workbook published ds", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "chartUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "viewsCount": 5 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent", + "title": "Timeline - Sent", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List", + "title": "Campaign List", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", + "title": "Summary", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Names)", + "schemaField": { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Values)", + "schemaField": { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign", + "title": "Mobile - Sent by Campaign", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", + "title": "Sheet 1", + "description": "", + "lastModified": { + "created": { + "time": 1639772911000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", + "title": "Sheet 2", + "description": "", + "lastModified": { + "created": { + "time": 1639773415000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)", + "schemaField": { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)", + "schemaField": { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)", + "schemaField": { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)", + "schemaField": { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)", + "schemaField": { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3", + "title": "Sheet 3", + "description": "", + "lastModified": { + "created": { + "time": 1640375456000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:TagSheet3" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)", + "schemaField": { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)", + "schemaField": { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)", + "schemaField": { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests", + "title": "Opened Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)", + "schemaField": { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)", + "schemaField": { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change", + "title": "Top 10 Items by Requests and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems", + "title": "Opened Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)", + "schemaField": { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)", + "schemaField": { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", + "title": "Overdue", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems", + "title": "High and Critical Priority Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)", + "schemaField": { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change", + "title": "Total Incidents by Category and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors", + "title": "Known Errors", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)", + "schemaField": { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "schemaField": { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)", + "schemaField": { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "schemaField": { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests", + "title": "Overdue Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident", + "title": "AVG Time to Solve an Incident", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)", + "schemaField": { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)", + "schemaField": { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F", + "title": "Made SLA?", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)", + "schemaField": { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "schemaField": { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", + "title": "Tooltip - Incident Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems", + "title": "Overdue Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", + "title": "Tooltip - Problem Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", + "title": "Tooltip - Request Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems", + "title": "Age of Active Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)", + "schemaField": { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)", + "schemaField": { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents", + "title": "Opened Incidents", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)", + "schemaField": { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)", + "schemaField": { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1", + "title": "published sheet ds", + "description": "", + "lastModified": { + "created": { + "time": 1641951867000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642658093000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)", + "schemaField": { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)", + "schemaField": { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "dashboardUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "viewsCount": 3 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": { + "luid": "fc9ea488-f810-4fa8-ac19-aa96018b5d66" + }, + "title": "Email Performance by Campaign", + "description": "", + "charts": [ + "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "dvd Rental Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Story 1", + "description": "", + "charts": [], + "datasets": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Executive Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Activity_Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Bounceback)" + ], + "transformOperation": "CalculatedFieldformula: [Sent Email] - [Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_Run_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign Run ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Choice_Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Choice Number %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Click Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)" + ], + "transformOperation": "CalculatedFieldformula: [Clickthrough Emails]/[Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Email Delivered)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Email Delivered)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Delivered Email]/[Sent Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Has_Predictive)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Is Predictive)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Mailing_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Mailing ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Non Clickthrough Email)" + ], + "transformOperation": "CalculatedFieldformula: [Delivered Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Opened Email]/[Delivered Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Open Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Non Clicked Emails)" + ], + "transformOperation": "CalculatedFieldformula: [Opened Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Platform %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),updatedAt)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Updated At)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),User_Agent)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),User Agent %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),programName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Workspace Name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "Marketo", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workspace Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Non Clicked Emails", + "nullable": false, + "description": "formula: [Opened Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Clickthrough Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Opened Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Opened Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Bounceback", + "nullable": false, + "description": "formula: [Sent Email] - [Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Customer Payment Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "actor+ (dvdrental)", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Address2", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "District", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "country, city", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Phone", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update (Address)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Actor Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "actor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),City)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Country/Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Discount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Discount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Location)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Manufacturer)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID %28Returns%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID %28Returns%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Orders)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Orders)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),People)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Postal Code)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit %28bin%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit Ratio)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Quantity)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Quantity)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region %28People%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region %28People%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Regional Manager)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Regional Manager)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returned)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returned)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returns)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returns)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Row ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Row ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sales)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Mode)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Mode)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),State)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sub-Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Top Customers by Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Regional Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T19:35:39Z", + "extractLastUpdateTime": "2018-01-18T19:35:39Z" + }, + "name": "Requests", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Closed by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Meta", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered item link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible elsewhere", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Update name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Special instructions", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Execution Plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Bundles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No search", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "List Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Guide", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Package", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Estimated Delivery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fulfillment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Backordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ignore price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Replace on upgrade", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Context", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Protection policy", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Omit price in cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalogs", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Entitlement script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published version", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Preview link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Guides", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item) 1", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve Time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Icon", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Display name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No proceed checkout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Use cart layout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order now", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Application", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Template", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalog", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Image", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workflow", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created from item design", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Hide price (mobile listings)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Availability", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "transformOperation": "CalculatedFieldformula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:21:33Z", + "extractLastUpdateTime": "2018-01-18T20:21:33Z" + }, + "name": "Problems", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "SLA due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Default assignee", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workaround", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u_", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Exclude manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lucha", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group email", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Include members", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)" + ], + "transformOperation": "CalculatedFieldformula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:13:08Z", + "extractLastUpdateTime": "2018-01-18T20:13:08Z" + }, + "name": "Incidents", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Attributes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close code (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Notify (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requires verification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Maintenance schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Warranty expiration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "GL account", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First discovered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Skip sync", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "DNS Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caller (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Department", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Managed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "PO number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Child Incidents (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "IP Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset tag", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked out", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fully qualified domain name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Installed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Purchased", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lease contract", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent Incident (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost currency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Monitor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Serial number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Owned by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Invoice number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order received", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discovery source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Operational status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reopen count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Most recent discovery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fault count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caused by Change (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "MAC Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Supported by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Support group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Justification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change Request (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Incident state (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Severity (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Can Print", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),Custom SQL Query)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),Custom SQL Query)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_id)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),payment_date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "New DataSource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:tag on published datasource" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "description": "description for test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "transformOperation": "GroupField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "transformOperation": "BinField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "transformOperation": "CalculatedFieldformula: SUM([Profit])/SUM([Sales])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "transformOperation": "SetField", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/Samples" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "description": "Description for Superstore dataset", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:SETFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "formula: SUM([Profit])/SUM([Sales])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:BINFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Person", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:GROUPFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Dvdrental Workbook/Customer Payment Query" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/test publish datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n ON p.customer_id = c.customer_id\nINNER JOIN staff s \n ON p.staff_id = s.staff_id\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "seller_city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/SubProject1/AbcJoinWorkbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "SubProject1" + }, + { + "id": "AbcJoinWorkbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "programName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "programId", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workspaceName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "updatedAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "postal_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "phone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "city_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I2", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "district", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "actor_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Person", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Returned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Product ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Problems", + "/prod/tableau/Acryl/default/Executive Dashboard/Requests", + "/prod/tableau/Acryl/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_for", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "special_instructions", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "configuration_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_catalog", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "estimated_delivery", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "context", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cat_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "backordered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sc_catalogs", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workflow", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_customer_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_standalone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_scope", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "template", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_proceed_checkout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "meta", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ordered_item_link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_version", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "image", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_policy", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "list_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order_now", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_item_staging", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "entitlement_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "icon", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ignore_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_closed", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_package", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_replace_on_upgrade", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_FLOAT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "use_sc_layout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "availability", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "custom_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_hide_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_time", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_search", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_bundle", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_update_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "preview", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "omit_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "u_u", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "exclude_manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lucha", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lu2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "default_assignee", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "include_members", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "related_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_around", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "known_error", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "severity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent_incident", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caller_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "child_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "incident_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "notify", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caused_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reopen_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "first_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "operational_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_cc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "attributes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "serial_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ip_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "support_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "supported_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "invoice_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "managed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fault_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "justification", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lease_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "monitor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "can_print", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "discovery_source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fqdn", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warranty_expiration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "owned_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset_tag", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manufacturer", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "purchase_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "department", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_out", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "unverified", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "skip_sync", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "po_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "gl_account", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "maintenance_schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dns_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mac_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "change_control", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Customer Payment Query", + "/prod/tableau/Acryl/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Customer Payment Query", + "/prod/tableau/Acryl/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:549dde2661c7ba205625767572f2d344", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "site_id": "190a6a5c-63ed-4de1-8045-site2" + }, + "name": "Site 2" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:549dde2661c7ba205625767572f2d344", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:549dde2661c7ba205625767572f2d344", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:549dde2661c7ba205625767572f2d344", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Site" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:549dde2661c7ba205625767572f2d344", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "190a6a5c-63ed-4de1-8045-faeae5df5b01" + }, + "name": "default" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:549dde2661c7ba205625767572f2d344" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "c30aafe5-44f4-4f28-80d3-d181010a263c" + }, + "name": "Project 2" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:549dde2661c7ba205625767572f2d344" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "910733aa-2e95-4ac3-a2e8-71570751099d" + }, + "name": "Samples" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:549dde2661c7ba205625767572f2d344" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "1f15d897-7f0c-7c59-037a-afa6a9b7c9a9" + }, + "externalUrl": "https://do-not-connect/#/site/site2/workbooks/15995", + "name": "Email Performance by Campaign", + "description": "Description for Email Performance by Campaign" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "661fabd0-bed6-8610-e066-0694a81a6cea" + }, + "externalUrl": "https://do-not-connect/#/site/site2/workbooks/15619", + "name": "Dvdrental Workbook", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:TagSheet3" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "6ffa5a7f-d852-78f1-6c6d-20ac23610ebf" + }, + "externalUrl": "https://do-not-connect/#/site/site2/workbooks/15605", + "name": "Executive Dashboard", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "bd040833-8f66-22c0-1b51-bd4ccf5eef7c" + }, + "externalUrl": "https://do-not-connect/#/site/site2/workbooks/17904", + "name": "Workbook published ds", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "chartUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "viewsCount": 5 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783" + }, + "externalUrl": "https://do-not-connect/t/site2/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent", + "title": "Timeline - Sent", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List", + "title": "Campaign List", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", + "title": "Summary", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Names)", + "schemaField": { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Values)", + "schemaField": { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign", + "title": "Mobile - Sent by Campaign", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/site2/views/dvdrental/Sheet1", + "title": "Sheet 1", + "description": "", + "lastModified": { + "created": { + "time": 1639772911000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/site2/views/dvdrental/Sheet2", + "title": "Sheet 2", + "description": "", + "lastModified": { + "created": { + "time": 1639773415000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)", + "schemaField": { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)", + "schemaField": { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)", + "schemaField": { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)", + "schemaField": { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)", + "schemaField": { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/site2/views/dvdrental/Sheet3", + "title": "Sheet 3", + "description": "", + "lastModified": { + "created": { + "time": 1640375456000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:TagSheet3" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)", + "schemaField": { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)", + "schemaField": { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)", + "schemaField": { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests", + "title": "Opened Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)", + "schemaField": { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)", + "schemaField": { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change", + "title": "Top 10 Items by Requests and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems", + "title": "Opened Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)", + "schemaField": { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)", + "schemaField": { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", + "title": "Overdue", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems", + "title": "High and Critical Priority Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)", + "schemaField": { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change", + "title": "Total Incidents by Category and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors", + "title": "Known Errors", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)", + "schemaField": { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "schemaField": { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)", + "schemaField": { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "schemaField": { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests", + "title": "Overdue Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident", + "title": "AVG Time to Solve an Incident", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)", + "schemaField": { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)", + "schemaField": { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F", + "title": "Made SLA?", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)", + "schemaField": { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "schemaField": { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/site2/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", + "title": "Tooltip - Incident Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems", + "title": "Overdue Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/site2/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", + "title": "Tooltip - Problem Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/site2/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", + "title": "Tooltip - Request Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems", + "title": "Age of Active Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)", + "schemaField": { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)", + "schemaField": { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents", + "title": "Opened Incidents", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)", + "schemaField": { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)", + "schemaField": { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/site2/views/Workbookpublishedds/Sheet1", + "title": "published sheet ds", + "description": "", + "lastModified": { + "created": { + "time": 1641951867000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642658093000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)", + "schemaField": { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)", + "schemaField": { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "dashboardUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "viewsCount": 3 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": { + "luid": "fc9ea488-f810-4fa8-ac19-aa96018b5d66" + }, + "title": "Email Performance by Campaign", + "description": "", + "charts": [ + "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/site2/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "dvd Rental Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/site2/views/dvdrental/dvdRentalDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Story 1", + "description": "", + "charts": [], + "datasets": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/site2/views/dvdrental/Story1" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Executive Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/site2/views/ExecutiveDashboard/ExecutiveDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Activity_Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Bounceback)" + ], + "transformOperation": "CalculatedFieldformula: [Sent Email] - [Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_Run_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign Run ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Choice_Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Choice Number %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Click Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)" + ], + "transformOperation": "CalculatedFieldformula: [Clickthrough Emails]/[Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Email Delivered)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Email Delivered)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Delivered Email]/[Sent Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Has_Predictive)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Is Predictive)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Mailing_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Mailing ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Non Clickthrough Email)" + ], + "transformOperation": "CalculatedFieldformula: [Delivered Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Opened Email]/[Delivered Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Open Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Non Clicked Emails)" + ], + "transformOperation": "CalculatedFieldformula: [Opened Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Platform %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),updatedAt)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Updated At)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),User_Agent)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),User Agent %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),programName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Workspace Name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "Marketo", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workspace Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Non Clicked Emails", + "nullable": false, + "description": "formula: [Opened Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Clickthrough Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Opened Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Opened Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Bounceback", + "nullable": false, + "description": "formula: [Sent Email] - [Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Customer Payment Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "actor+ (dvdrental)", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Address2", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "District", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "country, city", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Phone", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update (Address)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Actor Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "actor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),City)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Country/Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Discount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Discount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Location)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Manufacturer)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID %28Returns%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID %28Returns%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Orders)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Orders)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),People)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Postal Code)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit %28bin%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit Ratio)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Quantity)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Quantity)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region %28People%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region %28People%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Regional Manager)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Regional Manager)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returned)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returned)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returns)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returns)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Row ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Row ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sales)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Mode)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Mode)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),State)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sub-Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Top Customers by Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Regional Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T19:35:39Z", + "extractLastUpdateTime": "2018-01-18T19:35:39Z" + }, + "name": "Requests", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Closed by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Meta", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered item link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible elsewhere", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Update name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Special instructions", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Execution Plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Bundles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No search", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "List Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Guide", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Package", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Estimated Delivery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fulfillment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Backordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ignore price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Replace on upgrade", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Context", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Protection policy", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Omit price in cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalogs", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Entitlement script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published version", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Preview link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Guides", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item) 1", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve Time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Icon", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Display name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No proceed checkout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Use cart layout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order now", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Application", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Template", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalog", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Image", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workflow", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created from item design", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Hide price (mobile listings)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Availability", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "transformOperation": "CalculatedFieldformula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:21:33Z", + "extractLastUpdateTime": "2018-01-18T20:21:33Z" + }, + "name": "Problems", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "SLA due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Default assignee", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workaround", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u_", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Exclude manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lucha", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group email", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Include members", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)" + ], + "transformOperation": "CalculatedFieldformula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:13:08Z", + "extractLastUpdateTime": "2018-01-18T20:13:08Z" + }, + "name": "Incidents", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Attributes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close code (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Notify (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requires verification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Maintenance schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Warranty expiration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "GL account", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First discovered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Skip sync", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "DNS Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caller (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Department", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Managed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "PO number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Child Incidents (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "IP Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset tag", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked out", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fully qualified domain name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Installed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Purchased", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lease contract", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent Incident (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost currency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Monitor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Serial number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Owned by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Invoice number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order received", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discovery source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Operational status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reopen count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Most recent discovery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fault count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caused by Change (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "MAC Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Supported by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Support group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Justification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change Request (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Incident state (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Severity (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Can Print", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),Custom SQL Query)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),Custom SQL Query)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_id)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),payment_date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "New DataSource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:tag on published datasource" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "description": "description for test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "transformOperation": "GroupField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "transformOperation": "BinField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "transformOperation": "CalculatedFieldformula: SUM([Profit])/SUM([Sales])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "transformOperation": "SetField", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/Samples" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "description": "Description for Superstore dataset", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:SETFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "formula: SUM([Profit])/SUM([Sales])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:BINFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Person", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:GROUPFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Dvdrental Workbook/Customer Payment Query" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/test publish datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n ON p.customer_id = c.customer_id\nINNER JOIN staff s \n ON p.staff_id = s.staff_id\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "seller_city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/SubProject1/AbcJoinWorkbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "programName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "programId", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workspaceName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "updatedAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "postal_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "phone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "city_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I2", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "district", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "actor_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Person", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Returned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Product ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Problems", + "/prod/tableau/Site 2/default/Executive Dashboard/Requests", + "/prod/tableau/Site 2/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_for", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "special_instructions", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "configuration_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_catalog", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "estimated_delivery", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "context", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cat_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "backordered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sc_catalogs", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workflow", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_customer_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_standalone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_scope", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "template", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_proceed_checkout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "meta", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ordered_item_link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_version", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "image", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_policy", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "list_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order_now", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_item_staging", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "entitlement_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "icon", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ignore_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_closed", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_package", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_replace_on_upgrade", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_FLOAT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "use_sc_layout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "availability", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "custom_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_hide_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_time", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_search", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_bundle", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_update_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "preview", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "omit_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "u_u", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "exclude_manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lucha", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lu2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "default_assignee", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "include_members", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "related_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_around", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "known_error", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "severity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent_incident", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caller_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "child_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "incident_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "notify", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caused_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reopen_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "first_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "operational_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_cc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "attributes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "serial_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ip_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "support_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "supported_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "invoice_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "managed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fault_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "justification", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lease_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "monitor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "can_print", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "discovery_source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fqdn", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warranty_expiration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "owned_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset_tag", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manufacturer", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "purchase_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "department", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_out", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "unverified", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "skip_sync", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "po_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "gl_account", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "maintenance_schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dns_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mac_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "change_control", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Customer Payment Query", + "/prod/tableau/Site 2/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Customer Payment Query", + "/prod/tableau/Site 2/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:ATTRIBUTE", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "ATTRIBUTE" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:BINFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "BINFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:CALCULATEDFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "CALCULATEDFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:COLUMNFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "COLUMNFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:COUNT", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "COUNT" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:DATASOURCEFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "DATASOURCEFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:DIMENSION", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "DIMENSION" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:GROUPFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "GROUPFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:HIERARCHYFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "HIERARCHYFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:MEASURE", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "MEASURE" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:SETFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "SETFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:SUM", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "SUM" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:TagSheet3", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "TagSheet3" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:YEAR", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "YEAR" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:tag on published datasource", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "tag on published datasource" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json index 5e46b91c207a7..2aee4abc4d049 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json @@ -16,7 +16,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -32,7 +33,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -48,7 +50,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -66,7 +69,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -82,7 +86,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -102,7 +107,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -118,7 +124,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -134,7 +141,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -152,7 +160,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -168,7 +177,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -188,7 +198,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -204,7 +215,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -220,7 +232,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -238,7 +251,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -254,7 +268,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -274,7 +289,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -290,7 +306,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -306,7 +323,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -324,7 +342,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -340,7 +359,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -361,7 +381,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -383,7 +404,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -399,7 +421,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -415,7 +438,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -433,7 +457,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -449,6 +474,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -458,7 +484,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -474,7 +501,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -495,7 +523,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -517,7 +546,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -533,7 +563,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -549,7 +580,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -567,7 +599,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -583,6 +616,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -592,7 +626,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -612,7 +647,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -628,7 +664,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -649,7 +686,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -671,7 +709,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -687,7 +726,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -703,7 +743,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -721,7 +762,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -737,6 +779,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -746,7 +789,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -762,7 +806,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -783,7 +828,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -805,7 +851,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -821,7 +868,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -837,7 +885,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -855,7 +904,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -871,6 +921,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -880,7 +931,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -896,7 +948,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -917,7 +970,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -939,7 +993,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -955,7 +1010,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -971,7 +1027,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -989,7 +1046,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -1005,6 +1063,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1014,7 +1073,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -1030,7 +1090,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -1055,7 +1116,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -1076,7 +1138,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -1094,7 +1157,7 @@ "customProperties": { "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783" }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline - Sent", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent", "title": "Timeline - Sent", "description": "", "lastModified": { @@ -1129,6 +1192,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1141,7 +1205,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -1157,7 +1222,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -1330,7 +1396,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -1355,7 +1422,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -1371,7 +1439,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign List", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List", "title": "Campaign List", "description": "", "lastModified": { @@ -1406,6 +1474,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1418,7 +1487,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -1434,7 +1504,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -1893,7 +1964,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -1918,7 +1990,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -1969,6 +2042,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1981,7 +2055,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -1997,7 +2072,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -2508,7 +2584,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -2533,7 +2610,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -2549,7 +2627,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile - Sent by Campaign", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign", "title": "Mobile - Sent by Campaign", "description": "", "lastModified": { @@ -2584,6 +2662,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2596,7 +2675,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -2612,7 +2692,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3019,7 +3100,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3044,7 +3126,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3095,6 +3178,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3107,7 +3191,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3123,7 +3208,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3166,7 +3252,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3191,7 +3278,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3242,6 +3330,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3254,7 +3343,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3270,7 +3360,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3481,7 +3572,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3506,7 +3598,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3557,6 +3650,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3578,7 +3672,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3594,7 +3689,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3680,7 +3776,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3705,7 +3802,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3721,7 +3819,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Requests", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests", "title": "Opened Requests", "description": "", "lastModified": { @@ -3759,6 +3857,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3771,7 +3870,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -3787,7 +3887,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -4183,7 +4284,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -4208,7 +4310,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -4224,7 +4327,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top 10 Items by Requests and YoY Change", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change", "title": "Top 10 Items by Requests and YoY Change", "description": "", "lastModified": { @@ -4262,6 +4365,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4274,7 +4378,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -4290,7 +4395,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -4654,7 +4760,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -4679,7 +4786,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -4695,7 +4803,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems", "title": "Opened Problems", "description": "", "lastModified": { @@ -4733,6 +4841,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4745,7 +4854,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -4761,7 +4871,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -5157,7 +5268,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -5182,7 +5294,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -5233,6 +5346,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5245,7 +5359,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -5261,7 +5376,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -5541,7 +5657,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -5566,7 +5683,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -5582,7 +5700,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High and Critical Priority Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems", "title": "High and Critical Priority Problems", "description": "", "lastModified": { @@ -5620,6 +5738,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5632,7 +5751,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -5648,7 +5768,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -5905,7 +6026,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -5930,7 +6052,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -5946,7 +6069,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total Incidents by Category and YoY Change", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change", "title": "Total Incidents by Category and YoY Change", "description": "", "lastModified": { @@ -5981,6 +6104,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5993,7 +6117,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -6009,7 +6134,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -6292,7 +6418,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -6317,7 +6444,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -6333,7 +6461,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known Errors", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors", "title": "Known Errors", "description": "", "lastModified": { @@ -6371,6 +6499,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6383,7 +6512,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -6399,7 +6529,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -6734,7 +6865,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -6759,7 +6891,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -6775,7 +6908,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Requests", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests", "title": "Overdue Requests", "description": "", "lastModified": { @@ -6813,6 +6946,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6825,7 +6959,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -6841,7 +6976,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -7150,7 +7286,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -7175,7 +7312,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -7191,7 +7329,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG Time to Solve an Incident", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident", "title": "AVG Time to Solve an Incident", "description": "", "lastModified": { @@ -7226,6 +7364,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7238,7 +7377,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -7254,7 +7394,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -7482,7 +7623,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -7507,7 +7649,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -7523,7 +7666,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made SLA?", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F", "title": "Made SLA?", "description": "", "lastModified": { @@ -7561,6 +7704,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7573,7 +7717,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -7589,7 +7734,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -7846,7 +7992,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -7871,7 +8018,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -7922,6 +8070,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7934,7 +8083,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -7950,7 +8100,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -8100,7 +8251,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -8125,7 +8277,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -8141,7 +8294,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems", "title": "Overdue Problems", "description": "", "lastModified": { @@ -8179,6 +8332,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8191,7 +8345,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -8207,7 +8362,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -8542,7 +8698,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -8567,7 +8724,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -8621,6 +8779,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8633,7 +8792,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -8649,7 +8809,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -8880,7 +9041,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -8905,7 +9067,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -8959,6 +9122,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8971,7 +9135,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -8987,7 +9152,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -9244,7 +9410,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -9269,7 +9436,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -9285,7 +9453,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age of Active Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems", "title": "Age of Active Problems", "description": "", "lastModified": { @@ -9323,6 +9491,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9335,7 +9504,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -9351,7 +9521,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -9634,7 +9805,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -9659,7 +9831,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -9675,7 +9848,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Incidents", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents", "title": "Opened Incidents", "description": "", "lastModified": { @@ -9710,6 +9883,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9722,7 +9896,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -9738,7 +9913,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10076,7 +10252,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10101,7 +10278,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10152,6 +10330,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10164,7 +10343,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10180,7 +10360,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10266,7 +10447,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10291,7 +10473,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10312,7 +10495,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10367,6 +10551,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10379,7 +10564,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10395,7 +10581,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10420,7 +10607,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10470,6 +10658,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10482,7 +10671,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10498,7 +10688,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10523,7 +10714,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10571,6 +10763,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10583,7 +10776,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10599,7 +10793,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10624,7 +10819,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10686,6 +10882,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10698,7 +10895,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10714,7 +10912,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -10739,7 +10938,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -11083,7 +11283,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -11111,6 +11312,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -12874,7 +13076,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -12892,7 +13095,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -12908,7 +13112,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -12933,7 +13138,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -13034,7 +13240,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -13062,6 +13269,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -13277,7 +13485,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -13295,7 +13504,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -13311,7 +13521,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -13336,7 +13547,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -13369,7 +13581,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -13397,6 +13610,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -13784,7 +13998,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -13802,7 +14017,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -13818,7 +14034,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -13843,7 +14060,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -14278,7 +14496,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -14306,6 +14525,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -15028,7 +15248,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -15046,7 +15267,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -15062,7 +15284,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -15087,7 +15310,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -15233,7 +15457,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -15261,6 +15486,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -21851,7 +22077,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -21869,7 +22096,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -21885,7 +22113,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -21910,7 +22139,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -22099,7 +22329,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -22127,6 +22358,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -26003,7 +26235,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -26021,7 +26254,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -26037,7 +26271,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -26062,7 +26297,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -26213,7 +26449,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -26241,6 +26478,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31379,7 +31617,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -31397,7 +31636,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -31413,7 +31653,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -31438,7 +31679,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -31561,7 +31803,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -31589,6 +31832,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31791,7 +32035,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -31809,7 +32054,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -31825,7 +32071,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -31850,7 +32097,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -31899,7 +32147,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -31927,6 +32176,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -32019,7 +32269,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -32037,7 +32288,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -32053,7 +32305,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -32078,7 +32331,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -32182,7 +32436,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -32219,6 +32474,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -32467,7 +32723,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -32485,7 +32742,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -32501,7 +32759,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -32522,7 +32781,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -32656,7 +32916,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -32684,6 +32945,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -33536,7 +33798,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -33554,7 +33817,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -33570,7 +33834,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -33591,7 +33856,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -33607,7 +33873,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -33640,7 +33907,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -33775,7 +34043,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -33794,7 +34063,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -33819,7 +34089,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -33835,7 +34106,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -33876,7 +34148,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -34023,7 +34296,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -34042,7 +34316,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -34063,7 +34338,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -34143,7 +34419,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -34162,7 +34439,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -34326,7 +34604,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -34352,7 +34631,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -34588,7 +34868,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -34614,7 +34895,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -34826,7 +35108,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -34852,7 +35135,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -35016,7 +35300,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -35042,7 +35327,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -35194,7 +35480,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -35220,7 +35507,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -35360,7 +35648,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -35386,7 +35675,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -35478,7 +35768,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -35504,7 +35795,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -35572,7 +35864,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -35595,7 +35888,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -35663,7 +35957,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -35686,7 +35981,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -35982,7 +36278,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -36005,7 +36302,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -36795,7 +37093,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -36821,7 +37120,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -37705,7 +38005,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -37731,7 +38032,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -38687,7 +38989,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -38713,7 +39016,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -39537,7 +39841,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -39563,7 +39868,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -39871,7 +40177,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -39897,7 +40204,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -40745,7 +41053,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -40771,7 +41080,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -41763,7 +42073,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -41789,7 +42100,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42649,7 +42961,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42675,7 +42988,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42697,7 +43011,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42720,7 +43035,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42742,7 +43058,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42765,7 +43082,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42786,7 +43104,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42809,7 +43128,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42825,7 +43145,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42841,7 +43162,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42857,7 +43179,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42873,7 +43196,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42889,7 +43213,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42905,7 +43230,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42921,7 +43247,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42937,7 +43264,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42953,7 +43281,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42969,7 +43298,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -42985,7 +43315,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43001,7 +43332,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43017,7 +43349,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43033,7 +43366,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43049,7 +43383,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43065,7 +43400,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43081,7 +43417,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43097,7 +43434,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43113,7 +43451,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43129,7 +43468,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43145,7 +43485,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43161,7 +43502,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43177,7 +43519,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43193,7 +43536,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43209,7 +43553,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43225,7 +43570,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43241,7 +43587,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43257,7 +43604,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43273,7 +43621,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43289,7 +43638,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43305,7 +43655,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43321,7 +43672,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43337,7 +43689,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43353,7 +43706,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43369,7 +43723,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43385,7 +43740,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43401,7 +43757,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43417,7 +43774,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43433,7 +43791,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43449,7 +43808,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43465,7 +43825,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43481,7 +43842,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43497,7 +43859,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43513,7 +43876,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43529,7 +43893,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43545,7 +43910,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43561,7 +43927,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43577,7 +43944,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43593,7 +43961,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43609,7 +43978,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43625,7 +43995,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43641,7 +44012,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43657,7 +44029,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43673,7 +44046,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43689,7 +44063,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43705,7 +44080,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43721,7 +44097,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43737,7 +44114,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43753,7 +44131,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43769,7 +44148,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43785,7 +44165,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43801,7 +44182,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43817,7 +44199,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43833,7 +44216,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43849,7 +44233,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43865,7 +44250,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43881,7 +44267,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43897,7 +44284,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43913,7 +44301,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43929,7 +44318,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43945,7 +44335,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43961,7 +44352,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43977,7 +44369,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -43993,7 +44386,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -44009,7 +44403,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -44025,7 +44420,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -44041,7 +44437,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } }, { @@ -44057,1726 +44454,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", - "urn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", - "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Dvdrental Workbook" - }, - { - "id": "actor+ (dvdrental)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Dvdrental Workbook" - }, - { - "id": "actor+ (dvdrental)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Incidents" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Incidents" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Customer Payment Query" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Customer Payment Query" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "test publish datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_project_hierarchy" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json index c5417a8d212bc..53d4a802541c3 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json @@ -16,7 +16,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -32,7 +33,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -48,7 +50,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -66,7 +69,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -82,7 +86,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -102,7 +107,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -118,7 +124,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -134,7 +141,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -152,7 +160,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -168,7 +177,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -188,7 +198,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -204,7 +215,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -220,7 +232,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -238,7 +251,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -254,7 +268,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -276,7 +291,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -292,7 +308,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -308,7 +325,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -326,7 +344,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -342,6 +361,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -351,7 +371,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -367,7 +388,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -388,7 +410,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -410,7 +433,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -426,7 +450,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -442,7 +467,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -460,7 +486,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -476,6 +503,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -485,7 +513,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -505,7 +534,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -521,7 +551,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -542,7 +573,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -564,7 +596,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -580,7 +613,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -596,7 +630,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -614,7 +649,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -630,6 +666,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -639,7 +676,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -655,7 +693,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -676,7 +715,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -698,7 +738,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -714,7 +755,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -730,7 +772,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -748,7 +791,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -764,6 +808,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -773,7 +818,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -789,7 +835,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -810,7 +857,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -831,7 +879,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -849,7 +898,7 @@ "customProperties": { "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783" }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline - Sent", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent", "title": "Timeline - Sent", "description": "", "lastModified": { @@ -884,6 +933,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -896,7 +946,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -912,7 +963,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -1085,7 +1137,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -1110,7 +1163,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -1126,7 +1180,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign List", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List", "title": "Campaign List", "description": "", "lastModified": { @@ -1161,6 +1215,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1173,7 +1228,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -1189,7 +1245,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -1648,7 +1705,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -1673,7 +1731,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -1724,6 +1783,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1736,7 +1796,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -1752,7 +1813,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -2263,7 +2325,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -2288,7 +2351,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -2304,7 +2368,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile - Sent by Campaign", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign", "title": "Mobile - Sent by Campaign", "description": "", "lastModified": { @@ -2339,6 +2403,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2351,7 +2416,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -2367,7 +2433,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -2774,7 +2841,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -2799,7 +2867,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -2850,6 +2919,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2862,7 +2932,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -2878,7 +2949,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -2921,7 +2993,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -2946,7 +3019,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -2997,6 +3071,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3009,7 +3084,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -3025,7 +3101,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -3236,7 +3313,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -3261,7 +3339,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -3312,6 +3391,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3333,7 +3413,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -3349,7 +3430,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -3435,7 +3517,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -3460,7 +3543,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -3476,7 +3560,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Requests", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests", "title": "Opened Requests", "description": "", "lastModified": { @@ -3514,6 +3598,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3526,7 +3611,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -3542,7 +3628,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -3938,7 +4025,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -3963,7 +4051,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -3979,7 +4068,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top 10 Items by Requests and YoY Change", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change", "title": "Top 10 Items by Requests and YoY Change", "description": "", "lastModified": { @@ -4017,6 +4106,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4029,7 +4119,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -4045,7 +4136,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -4409,7 +4501,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -4434,7 +4527,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -4450,7 +4544,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems", "title": "Opened Problems", "description": "", "lastModified": { @@ -4488,6 +4582,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4500,7 +4595,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -4516,7 +4612,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -4912,7 +5009,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -4937,7 +5035,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -4988,6 +5087,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5000,7 +5100,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -5016,7 +5117,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -5296,7 +5398,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -5321,7 +5424,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -5337,7 +5441,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High and Critical Priority Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems", "title": "High and Critical Priority Problems", "description": "", "lastModified": { @@ -5375,6 +5479,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5387,7 +5492,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -5403,7 +5509,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -5660,7 +5767,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -5685,7 +5793,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -5701,7 +5810,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total Incidents by Category and YoY Change", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change", "title": "Total Incidents by Category and YoY Change", "description": "", "lastModified": { @@ -5736,6 +5845,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5748,7 +5858,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -5764,7 +5875,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -6047,7 +6159,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -6072,7 +6185,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -6088,7 +6202,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known Errors", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors", "title": "Known Errors", "description": "", "lastModified": { @@ -6126,6 +6240,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6138,7 +6253,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -6154,7 +6270,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -6489,7 +6606,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -6514,7 +6632,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -6530,7 +6649,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Requests", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests", "title": "Overdue Requests", "description": "", "lastModified": { @@ -6568,6 +6687,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6580,7 +6700,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -6596,7 +6717,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -6905,7 +7027,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -6930,7 +7053,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -6946,7 +7070,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG Time to Solve an Incident", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident", "title": "AVG Time to Solve an Incident", "description": "", "lastModified": { @@ -6981,6 +7105,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6993,7 +7118,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -7009,7 +7135,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -7237,7 +7364,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -7262,7 +7390,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -7278,7 +7407,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made SLA?", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F", "title": "Made SLA?", "description": "", "lastModified": { @@ -7316,6 +7445,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7328,7 +7458,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -7344,7 +7475,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -7601,7 +7733,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -7626,7 +7759,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -7677,6 +7811,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7689,7 +7824,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -7705,7 +7841,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -7855,7 +7992,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -7880,7 +8018,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -7896,7 +8035,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems", "title": "Overdue Problems", "description": "", "lastModified": { @@ -7934,6 +8073,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7946,7 +8086,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -7962,7 +8103,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -8297,7 +8439,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -8322,7 +8465,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -8376,6 +8520,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8388,7 +8533,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -8404,7 +8550,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -8635,7 +8782,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -8660,7 +8808,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -8714,6 +8863,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8726,7 +8876,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -8742,7 +8893,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -8999,7 +9151,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -9024,7 +9177,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -9040,7 +9194,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age of Active Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems", "title": "Age of Active Problems", "description": "", "lastModified": { @@ -9078,6 +9232,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9090,7 +9245,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -9106,7 +9262,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -9389,7 +9546,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -9414,7 +9572,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -9430,7 +9589,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Incidents", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents", "title": "Opened Incidents", "description": "", "lastModified": { @@ -9465,6 +9624,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9477,7 +9637,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -9493,7 +9654,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -9831,7 +9993,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -9856,7 +10019,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -9907,6 +10071,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9919,7 +10084,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -9935,7 +10101,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10021,7 +10188,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10046,7 +10214,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10067,7 +10236,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10122,6 +10292,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10134,7 +10305,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10150,7 +10322,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10175,7 +10348,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10225,6 +10399,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10237,7 +10412,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10253,7 +10429,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10278,7 +10455,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10326,6 +10504,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10338,7 +10517,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10354,7 +10534,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10379,7 +10560,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10441,6 +10623,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10453,7 +10636,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10469,7 +10653,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10494,7 +10679,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10838,7 +11024,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -10866,6 +11053,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -12629,7 +12817,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -12647,7 +12836,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -12663,7 +12853,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -12688,7 +12879,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -12789,7 +12981,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -12817,6 +13010,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -13032,7 +13226,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -13050,7 +13245,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -13066,7 +13262,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -13091,7 +13288,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -13124,7 +13322,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -13152,6 +13351,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -13539,7 +13739,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -13557,7 +13758,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -13573,7 +13775,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -13598,7 +13801,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -14033,7 +14237,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -14061,6 +14266,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -14783,7 +14989,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -14801,7 +15008,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -14817,7 +15025,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -14842,7 +15051,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -14988,7 +15198,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -15016,6 +15227,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -21606,7 +21818,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -21624,7 +21837,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -21640,7 +21854,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -21665,7 +21880,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -21854,7 +22070,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -21882,6 +22099,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -25758,7 +25976,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -25776,7 +25995,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -25792,7 +26012,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -25817,7 +26038,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -25968,7 +26190,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -25996,6 +26219,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31134,7 +31358,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -31152,7 +31377,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -31168,7 +31394,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -31193,7 +31420,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -31316,7 +31544,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -31344,6 +31573,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31546,7 +31776,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -31564,7 +31795,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -31580,7 +31812,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -31605,7 +31838,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -31654,7 +31888,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -31682,6 +31917,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31774,7 +32010,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -31792,7 +32029,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -31808,7 +32046,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -31833,7 +32072,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -31937,7 +32177,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -31974,6 +32215,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -32222,7 +32464,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -32240,7 +32483,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -32256,7 +32500,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -32277,7 +32522,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -32411,7 +32657,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -32439,6 +32686,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -33291,7 +33539,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33309,7 +33558,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33325,7 +33575,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33346,7 +33597,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33362,7 +33614,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33395,7 +33648,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33530,7 +33784,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33549,7 +33804,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33574,7 +33830,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33590,7 +33847,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33631,7 +33889,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33778,7 +34037,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33797,7 +34057,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33818,7 +34079,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33905,7 +34167,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33924,7 +34187,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -33947,7 +34211,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -34111,7 +34376,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -34137,7 +34403,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -34373,7 +34640,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -34399,7 +34667,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -34611,7 +34880,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -34637,7 +34907,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -34801,7 +35072,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -34827,7 +35099,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -34979,7 +35252,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -35005,7 +35279,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -35145,7 +35420,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -35171,7 +35447,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -35263,7 +35540,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -35289,7 +35567,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -35357,7 +35636,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -35380,7 +35660,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -35448,7 +35729,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -35471,7 +35753,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -35767,7 +36050,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -35790,7 +36074,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -36580,7 +36865,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -36606,7 +36892,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -37490,7 +37777,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -37516,7 +37804,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -38472,7 +38761,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -38498,7 +38788,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -39322,7 +39613,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -39348,7 +39640,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -39656,7 +39949,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -39682,7 +39976,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -40530,7 +40825,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -40556,7 +40852,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -41548,7 +41845,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -41574,7 +41872,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42434,7 +42733,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42460,7 +42760,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42482,7 +42783,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42505,7 +42807,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42527,7 +42830,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42550,7 +42854,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42571,7 +42876,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42594,7 +42900,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42610,7 +42917,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42626,7 +42934,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42642,7 +42951,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42658,7 +42968,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42674,7 +42985,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42690,7 +43002,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42706,7 +43019,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42722,7 +43036,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42738,7 +43053,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42754,7 +43070,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42770,7 +43087,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42786,7 +43104,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42802,7 +43121,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42818,7 +43138,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42834,7 +43155,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42850,7 +43172,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42866,7 +43189,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42882,7 +43206,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42898,7 +43223,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42914,7 +43240,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42930,7 +43257,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42946,7 +43274,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42962,7 +43291,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42978,7 +43308,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -42994,7 +43325,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43010,7 +43342,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43026,7 +43359,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43042,7 +43376,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43058,7 +43393,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43074,7 +43410,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43090,7 +43427,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43106,7 +43444,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43122,7 +43461,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43138,7 +43478,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43154,7 +43495,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43170,7 +43512,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43186,7 +43529,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43202,7 +43546,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43218,7 +43563,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43234,7 +43580,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43250,7 +43597,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43266,7 +43614,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43282,7 +43631,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43298,7 +43648,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43314,7 +43665,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43330,7 +43682,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43346,7 +43699,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43362,7 +43716,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43378,7 +43733,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43394,7 +43750,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43410,7 +43767,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43426,7 +43784,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43442,7 +43801,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43458,7 +43818,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43474,7 +43835,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43490,7 +43852,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43506,7 +43869,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43522,7 +43886,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43538,7 +43903,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43554,7 +43920,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43570,7 +43937,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43586,7 +43954,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43602,7 +43971,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43618,7 +43988,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43634,7 +44005,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43650,7 +44022,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43666,7 +44039,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43682,7 +44056,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43698,7 +44073,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43714,7 +44090,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43730,7 +44107,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43746,7 +44124,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43762,7 +44141,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43778,7 +44158,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43794,7 +44175,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43810,7 +44192,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43826,7 +44209,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } }, { @@ -43842,1703 +44226,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", - "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - }, - { - "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", - "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "SubProject1" - }, - { - "id": "AbcJoinWorkbook" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Dvdrental Workbook" - }, - { - "id": "actor+ (dvdrental)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Dvdrental Workbook" - }, - { - "id": "actor+ (dvdrental)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Incidents" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Incidents" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Customer Payment Query" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "Customer Payment Query" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "default" - }, - { - "id": "test publish datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_signout_timeout" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_site_name_pattern_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_site_name_pattern_mces_golden.json new file mode 100644 index 0000000000000..4811e462acdb2 --- /dev/null +++ b/metadata-ingestion/tests/integration/tableau/tableau_site_name_pattern_mces_golden.json @@ -0,0 +1,44636 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:549dde2661c7ba205625767572f2d344", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "site_id": "190a6a5c-63ed-4de1-8045-site2" + }, + "name": "Site 2" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:549dde2661c7ba205625767572f2d344", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:549dde2661c7ba205625767572f2d344", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:549dde2661c7ba205625767572f2d344", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Site" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:549dde2661c7ba205625767572f2d344", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "190a6a5c-63ed-4de1-8045-faeae5df5b01" + }, + "name": "default" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:549dde2661c7ba205625767572f2d344" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "c30aafe5-44f4-4f28-80d3-d181010a263c" + }, + "name": "Project 2" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:549dde2661c7ba205625767572f2d344" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "910733aa-2e95-4ac3-a2e8-71570751099d" + }, + "name": "Samples" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:549dde2661c7ba205625767572f2d344" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "1f15d897-7f0c-7c59-037a-afa6a9b7c9a9" + }, + "externalUrl": "https://do-not-connect/#/site/site2/workbooks/15995", + "name": "Email Performance by Campaign", + "description": "Description for Email Performance by Campaign" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "661fabd0-bed6-8610-e066-0694a81a6cea" + }, + "externalUrl": "https://do-not-connect/#/site/site2/workbooks/15619", + "name": "Dvdrental Workbook", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:TagSheet3" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "6ffa5a7f-d852-78f1-6c6d-20ac23610ebf" + }, + "externalUrl": "https://do-not-connect/#/site/site2/workbooks/15605", + "name": "Executive Dashboard", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "bd040833-8f66-22c0-1b51-bd4ccf5eef7c" + }, + "externalUrl": "https://do-not-connect/#/site/site2/workbooks/17904", + "name": "Workbook published ds", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "chartUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "viewsCount": 5 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783" + }, + "externalUrl": "https://do-not-connect/t/site2/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent", + "title": "Timeline - Sent", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List", + "title": "Campaign List", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", + "title": "Summary", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Names)", + "schemaField": { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Values)", + "schemaField": { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign", + "title": "Mobile - Sent by Campaign", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/site2/views/dvdrental/Sheet1", + "title": "Sheet 1", + "description": "", + "lastModified": { + "created": { + "time": 1639772911000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/site2/views/dvdrental/Sheet2", + "title": "Sheet 2", + "description": "", + "lastModified": { + "created": { + "time": 1639773415000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)", + "schemaField": { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)", + "schemaField": { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)", + "schemaField": { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)", + "schemaField": { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)", + "schemaField": { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/site2/views/dvdrental/Sheet3", + "title": "Sheet 3", + "description": "", + "lastModified": { + "created": { + "time": 1640375456000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:TagSheet3" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)", + "schemaField": { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)", + "schemaField": { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)", + "schemaField": { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests", + "title": "Opened Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)", + "schemaField": { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)", + "schemaField": { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change", + "title": "Top 10 Items by Requests and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems", + "title": "Opened Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)", + "schemaField": { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)", + "schemaField": { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", + "title": "Overdue", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems", + "title": "High and Critical Priority Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)", + "schemaField": { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change", + "title": "Total Incidents by Category and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors", + "title": "Known Errors", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)", + "schemaField": { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "schemaField": { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)", + "schemaField": { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "schemaField": { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests", + "title": "Overdue Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident", + "title": "AVG Time to Solve an Incident", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)", + "schemaField": { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)", + "schemaField": { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F", + "title": "Made SLA?", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)", + "schemaField": { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "schemaField": { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/site2/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", + "title": "Tooltip - Incident Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems", + "title": "Overdue Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/site2/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", + "title": "Tooltip - Problem Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/site2/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", + "title": "Tooltip - Request Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems", + "title": "Age of Active Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)", + "schemaField": { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)", + "schemaField": { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/site2/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents", + "title": "Opened Incidents", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)", + "schemaField": { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)", + "schemaField": { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/site2/views/Workbookpublishedds/Sheet1", + "title": "published sheet ds", + "description": "", + "lastModified": { + "created": { + "time": 1641951867000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642658093000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)", + "schemaField": { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)", + "schemaField": { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "dashboardUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "viewsCount": 3 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": { + "luid": "fc9ea488-f810-4fa8-ac19-aa96018b5d66" + }, + "title": "Email Performance by Campaign", + "description": "", + "charts": [ + "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/site2/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "dvd Rental Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/site2/views/dvdrental/dvdRentalDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Story 1", + "description": "", + "charts": [], + "datasets": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/site2/views/dvdrental/Story1" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Executive Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/site2/views/ExecutiveDashboard/ExecutiveDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Activity_Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Bounceback)" + ], + "transformOperation": "CalculatedFieldformula: [Sent Email] - [Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_Run_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign Run ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Choice_Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Choice Number %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Click Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)" + ], + "transformOperation": "CalculatedFieldformula: [Clickthrough Emails]/[Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Email Delivered)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Email Delivered)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Delivered Email]/[Sent Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Has_Predictive)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Is Predictive)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Mailing_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Mailing ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Non Clickthrough Email)" + ], + "transformOperation": "CalculatedFieldformula: [Delivered Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Opened Email]/[Delivered Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Open Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Non Clicked Emails)" + ], + "transformOperation": "CalculatedFieldformula: [Opened Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Platform %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),updatedAt)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Updated At)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),User_Agent)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),User Agent %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),programName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Workspace Name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "Marketo", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workspace Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Non Clicked Emails", + "nullable": false, + "description": "formula: [Opened Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Clickthrough Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Opened Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Opened Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Bounceback", + "nullable": false, + "description": "formula: [Sent Email] - [Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Customer Payment Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "actor+ (dvdrental)", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Address2", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "District", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "country, city", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Phone", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update (Address)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Actor Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "actor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),City)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Country/Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Discount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Discount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Location)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Manufacturer)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID %28Returns%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID %28Returns%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Orders)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Orders)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),People)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Postal Code)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit %28bin%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit Ratio)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Quantity)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Quantity)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region %28People%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region %28People%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Regional Manager)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Regional Manager)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returned)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returned)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returns)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returns)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Row ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Row ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sales)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Mode)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Mode)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),State)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sub-Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Top Customers by Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Regional Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T19:35:39Z", + "extractLastUpdateTime": "2018-01-18T19:35:39Z" + }, + "name": "Requests", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Closed by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Meta", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered item link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible elsewhere", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Update name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Special instructions", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Execution Plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Bundles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No search", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "List Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Guide", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Package", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Estimated Delivery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fulfillment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Backordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ignore price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Replace on upgrade", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Context", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Protection policy", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Omit price in cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalogs", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Entitlement script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published version", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Preview link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Guides", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item) 1", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve Time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Icon", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Display name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No proceed checkout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Use cart layout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order now", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Application", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Template", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalog", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Image", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workflow", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created from item design", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Hide price (mobile listings)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Availability", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "transformOperation": "CalculatedFieldformula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:21:33Z", + "extractLastUpdateTime": "2018-01-18T20:21:33Z" + }, + "name": "Problems", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "SLA due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Default assignee", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workaround", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u_", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Exclude manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lucha", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group email", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Include members", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)" + ], + "transformOperation": "CalculatedFieldformula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:13:08Z", + "extractLastUpdateTime": "2018-01-18T20:13:08Z" + }, + "name": "Incidents", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Attributes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close code (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Notify (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requires verification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Maintenance schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Warranty expiration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "GL account", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First discovered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Skip sync", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "DNS Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caller (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Department", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Managed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "PO number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Child Incidents (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "IP Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset tag", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked out", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fully qualified domain name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Installed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Purchased", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lease contract", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent Incident (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost currency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Monitor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Serial number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Owned by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Invoice number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order received", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discovery source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Operational status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reopen count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Most recent discovery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fault count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caused by Change (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "MAC Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Supported by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Support group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Justification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change Request (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Incident state (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Severity (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Can Print", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),Custom SQL Query)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),Custom SQL Query)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_id)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),payment_date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "New DataSource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:tag on published datasource" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "description": "description for test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "transformOperation": "GroupField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "transformOperation": "BinField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "transformOperation": "CalculatedFieldformula: SUM([Profit])/SUM([Sales])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "transformOperation": "SetField", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/Samples" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "description": "Description for Superstore dataset", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:SETFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "formula: SUM([Profit])/SUM([Sales])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:BINFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Person", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:GROUPFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Dvdrental Workbook/Customer Payment Query" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/test publish datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n ON p.customer_id = c.customer_id\nINNER JOIN staff s \n ON p.staff_id = s.staff_id\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:549dde2661c7ba205625767572f2d344", + "urn": "urn:li:container:549dde2661c7ba205625767572f2d344" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "seller_city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/SubProject1/AbcJoinWorkbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "SubProject1" + }, + { + "id": "AbcJoinWorkbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "programName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "programId", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workspaceName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "updatedAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "postal_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "phone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "city_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I2", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "district", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "actor_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Person", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Returned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Product ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Problems", + "/prod/tableau/Site 2/default/Executive Dashboard/Requests", + "/prod/tableau/Site 2/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_for", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "special_instructions", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "configuration_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_catalog", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "estimated_delivery", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "context", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cat_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "backordered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sc_catalogs", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workflow", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_customer_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_standalone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_scope", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "template", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_proceed_checkout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "meta", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ordered_item_link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_version", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "image", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_policy", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "list_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order_now", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_item_staging", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "entitlement_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "icon", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ignore_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_closed", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_package", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_replace_on_upgrade", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_FLOAT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "use_sc_layout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "availability", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "custom_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_hide_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_time", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_search", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_bundle", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_update_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "preview", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "omit_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "u_u", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "exclude_manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lucha", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lu2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "default_assignee", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "include_members", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "related_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_around", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "known_error", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "severity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent_incident", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caller_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "child_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "incident_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "notify", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caused_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reopen_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "first_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "operational_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_cc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "attributes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "serial_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ip_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "support_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "supported_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "invoice_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "managed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fault_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "justification", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lease_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "monitor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "can_print", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "discovery_source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fqdn", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warranty_expiration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "owned_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset_tag", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manufacturer", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "purchase_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "department", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_out", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "unverified", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "skip_sync", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "po_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "gl_account", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "maintenance_schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dns_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mac_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "change_control", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Customer Payment Query", + "/prod/tableau/Site 2/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/Customer Payment Query", + "/prod/tableau/Site 2/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Site 2/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Site 2" + }, + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:ATTRIBUTE", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "ATTRIBUTE" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:BINFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "BINFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:CALCULATEDFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "CALCULATEDFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:COLUMNFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "COLUMNFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:COUNT", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "COUNT" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:DATASOURCEFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "DATASOURCEFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:DIMENSION", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "DIMENSION" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:GROUPFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "GROUPFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:HIERARCHYFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "HIERARCHYFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:MEASURE", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "MEASURE" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:SETFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "SETFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:SUM", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "SUM" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:TagSheet3", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "TagSheet3" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:YEAR", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "YEAR" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:tag on published datasource", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "tag on published datasource" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_site_name_pattern_ingest" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_sites_as_container_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_sites_as_container_mces_golden.json new file mode 100644 index 0000000000000..9c83fdbc1d879 --- /dev/null +++ b/metadata-ingestion/tests/integration/tableau/tableau_sites_as_container_mces_golden.json @@ -0,0 +1,44636 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "site_id": "190a6a5c-63ed-4de1-8045-site1" + }, + "name": "Acryl" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Site" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "190a6a5c-63ed-4de1-8045-faeae5df5b01" + }, + "name": "default" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "c30aafe5-44f4-4f28-80d3-d181010a263c" + }, + "name": "Project 2" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "910733aa-2e95-4ac3-a2e8-71570751099d" + }, + "name": "Samples" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "1f15d897-7f0c-7c59-037a-afa6a9b7c9a9" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15995", + "name": "Email Performance by Campaign", + "description": "Description for Email Performance by Campaign" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "661fabd0-bed6-8610-e066-0694a81a6cea" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15619", + "name": "Dvdrental Workbook", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:TagSheet3" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "6ffa5a7f-d852-78f1-6c6d-20ac23610ebf" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15605", + "name": "Executive Dashboard", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "bd040833-8f66-22c0-1b51-bd4ccf5eef7c" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/17904", + "name": "Workbook published ds", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "chartUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "viewsCount": 5 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent", + "title": "Timeline - Sent", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List", + "title": "Campaign List", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", + "title": "Summary", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Names)", + "schemaField": { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Values)", + "schemaField": { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign", + "title": "Mobile - Sent by Campaign", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", + "title": "Sheet 1", + "description": "", + "lastModified": { + "created": { + "time": 1639772911000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", + "title": "Sheet 2", + "description": "", + "lastModified": { + "created": { + "time": 1639773415000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)", + "schemaField": { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)", + "schemaField": { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)", + "schemaField": { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)", + "schemaField": { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)", + "schemaField": { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3", + "title": "Sheet 3", + "description": "", + "lastModified": { + "created": { + "time": 1640375456000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:TagSheet3" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)", + "schemaField": { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)", + "schemaField": { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)", + "schemaField": { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests", + "title": "Opened Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)", + "schemaField": { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)", + "schemaField": { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change", + "title": "Top 10 Items by Requests and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems", + "title": "Opened Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)", + "schemaField": { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)", + "schemaField": { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", + "title": "Overdue", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems", + "title": "High and Critical Priority Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)", + "schemaField": { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change", + "title": "Total Incidents by Category and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors", + "title": "Known Errors", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)", + "schemaField": { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "schemaField": { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)", + "schemaField": { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "schemaField": { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests", + "title": "Overdue Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident", + "title": "AVG Time to Solve an Incident", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)", + "schemaField": { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)", + "schemaField": { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F", + "title": "Made SLA?", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)", + "schemaField": { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "schemaField": { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", + "title": "Tooltip - Incident Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems", + "title": "Overdue Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", + "title": "Tooltip - Problem Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", + "title": "Tooltip - Request Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems", + "title": "Age of Active Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)", + "schemaField": { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)", + "schemaField": { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents", + "title": "Opened Incidents", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)", + "schemaField": { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)", + "schemaField": { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1", + "title": "published sheet ds", + "description": "", + "lastModified": { + "created": { + "time": 1641951867000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642658093000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)", + "schemaField": { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)", + "schemaField": { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "dashboardUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "viewsCount": 3 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": { + "luid": "fc9ea488-f810-4fa8-ac19-aa96018b5d66" + }, + "title": "Email Performance by Campaign", + "description": "", + "charts": [ + "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "dvd Rental Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Story 1", + "description": "", + "charts": [], + "datasets": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Executive Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Activity_Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Bounceback)" + ], + "transformOperation": "CalculatedFieldformula: [Sent Email] - [Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_Run_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign Run ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Choice_Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Choice Number %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Click Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)" + ], + "transformOperation": "CalculatedFieldformula: [Clickthrough Emails]/[Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Email Delivered)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Email Delivered)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Delivered Email]/[Sent Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Has_Predictive)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Is Predictive)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Mailing_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Mailing ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Non Clickthrough Email)" + ], + "transformOperation": "CalculatedFieldformula: [Delivered Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Opened Email]/[Delivered Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Open Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Non Clicked Emails)" + ], + "transformOperation": "CalculatedFieldformula: [Opened Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Platform %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),updatedAt)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Updated At)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),User_Agent)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),User Agent %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),programName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Workspace Name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "Marketo", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workspace Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Non Clicked Emails", + "nullable": false, + "description": "formula: [Opened Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Clickthrough Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Opened Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Opened Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Bounceback", + "nullable": false, + "description": "formula: [Sent Email] - [Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Customer Payment Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "actor+ (dvdrental)", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Address2", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "District", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "country, city", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Phone", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update (Address)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Actor Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "actor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),City)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Country/Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Discount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Discount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Location)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Manufacturer)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID %28Returns%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID %28Returns%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Orders)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Orders)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),People)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Postal Code)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit %28bin%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit Ratio)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Quantity)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Quantity)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region %28People%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region %28People%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Regional Manager)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Regional Manager)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returned)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returned)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returns)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returns)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Row ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Row ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sales)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Mode)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Mode)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),State)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sub-Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Top Customers by Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Regional Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T19:35:39Z", + "extractLastUpdateTime": "2018-01-18T19:35:39Z" + }, + "name": "Requests", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Closed by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Meta", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered item link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible elsewhere", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Update name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Special instructions", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Execution Plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Bundles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No search", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "List Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Guide", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Package", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Estimated Delivery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fulfillment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Backordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ignore price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Replace on upgrade", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Context", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Protection policy", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Omit price in cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalogs", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Entitlement script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published version", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Preview link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Guides", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item) 1", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve Time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Icon", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Display name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No proceed checkout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Use cart layout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order now", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Application", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Template", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalog", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Image", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workflow", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created from item design", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Hide price (mobile listings)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Availability", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "transformOperation": "CalculatedFieldformula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:21:33Z", + "extractLastUpdateTime": "2018-01-18T20:21:33Z" + }, + "name": "Problems", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "SLA due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Default assignee", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workaround", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u_", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Exclude manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lucha", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group email", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Include members", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)" + ], + "transformOperation": "CalculatedFieldformula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:13:08Z", + "extractLastUpdateTime": "2018-01-18T20:13:08Z" + }, + "name": "Incidents", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Attributes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close code (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Notify (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requires verification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Maintenance schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Warranty expiration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "GL account", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First discovered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Skip sync", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "DNS Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caller (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Department", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Managed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "PO number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Child Incidents (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "IP Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset tag", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked out", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fully qualified domain name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Installed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Purchased", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lease contract", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent Incident (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost currency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Monitor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Serial number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Owned by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Invoice number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order received", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discovery source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Operational status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reopen count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Most recent discovery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fault count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caused by Change (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "MAC Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Supported by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Support group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Justification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change Request (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Incident state (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Severity (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Can Print", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),Custom SQL Query)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),Custom SQL Query)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_id)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),payment_date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "New DataSource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:tag on published datasource" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "description": "description for test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "transformOperation": "GroupField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "transformOperation": "BinField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "transformOperation": "CalculatedFieldformula: SUM([Profit])/SUM([Sales])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "transformOperation": "SetField", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/Samples" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "description": "Description for Superstore dataset", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:SETFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "formula: SUM([Profit])/SUM([Sales])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:BINFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Person", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:GROUPFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Dvdrental Workbook/Customer Payment Query" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/test publish datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n ON p.customer_id = c.customer_id\nINNER JOIN staff s \n ON p.staff_id = s.staff_id\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc", + "urn": "urn:li:container:d60bc5427649a8dc937b4e2a510f84bc" + }, + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "seller_city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/SubProject1/AbcJoinWorkbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "SubProject1" + }, + { + "id": "AbcJoinWorkbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "programName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "programId", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workspaceName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "updatedAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "postal_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "phone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "city_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I2", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "district", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "actor_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Person", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Returned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Product ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Problems", + "/prod/tableau/Acryl/default/Executive Dashboard/Requests", + "/prod/tableau/Acryl/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_for", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "special_instructions", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "configuration_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_catalog", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "estimated_delivery", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "context", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cat_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "backordered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sc_catalogs", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workflow", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_customer_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_standalone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_scope", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "template", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_proceed_checkout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "meta", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ordered_item_link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_version", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "image", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_policy", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "list_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order_now", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_item_staging", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "entitlement_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "icon", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ignore_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_closed", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_package", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_replace_on_upgrade", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_FLOAT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "use_sc_layout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "availability", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "custom_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_hide_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_time", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_search", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_bundle", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_update_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "preview", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "omit_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "u_u", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "exclude_manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lucha", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lu2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "default_assignee", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "include_members", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "related_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_around", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "known_error", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "severity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent_incident", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caller_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "child_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "incident_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "notify", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caused_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reopen_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "first_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "operational_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_cc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "attributes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "serial_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ip_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "support_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "supported_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "invoice_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "managed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fault_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "justification", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lease_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "monitor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "can_print", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "discovery_source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fqdn", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warranty_expiration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "owned_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset_tag", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manufacturer", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "purchase_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "department", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_out", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "unverified", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "skip_sync", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "po_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "gl_account", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "maintenance_schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dns_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mac_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "change_control", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Customer Payment Query", + "/prod/tableau/Acryl/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/Customer Payment Query", + "/prod/tableau/Acryl/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Acryl/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Acryl" + }, + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:ATTRIBUTE", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "ATTRIBUTE" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:BINFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "BINFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:CALCULATEDFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "CALCULATEDFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:COLUMNFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "COLUMNFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:COUNT", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "COUNT" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:DATASOURCEFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "DATASOURCEFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:DIMENSION", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "DIMENSION" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:GROUPFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "GROUPFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:HIERARCHYFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "HIERARCHYFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:MEASURE", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "MEASURE" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:SETFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "SETFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:SUM", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "SUM" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:TagSheet3", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "TagSheet3" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:YEAR", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "YEAR" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:tag on published datasource", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "tag on published datasource" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_multiple_site_ingestion" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json index 5dc8fd9522da4..fdecf887b5305 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json @@ -17,7 +17,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -33,7 +34,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -50,7 +52,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -68,7 +71,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -89,7 +93,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -110,7 +115,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -126,7 +132,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -143,7 +150,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -161,7 +169,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -182,7 +191,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -205,7 +215,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -221,7 +232,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -238,7 +250,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -256,7 +269,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -272,6 +286,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -281,7 +296,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -297,7 +313,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -322,7 +339,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -345,7 +363,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -361,7 +380,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -378,7 +398,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -396,7 +417,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -412,6 +434,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -421,7 +444,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -441,7 +465,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -457,7 +482,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -482,7 +508,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -505,7 +532,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -521,7 +549,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -538,7 +567,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -556,7 +586,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -572,6 +603,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -581,7 +613,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -597,7 +630,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -622,7 +656,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -645,7 +680,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -661,7 +697,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -678,7 +715,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -696,7 +734,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -712,6 +751,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -721,7 +761,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -737,7 +778,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -762,7 +804,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -783,7 +826,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -802,7 +846,7 @@ "customProperties": { "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783" }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline - Sent", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent", "title": "Timeline - Sent", "description": "", "lastModified": { @@ -837,6 +881,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -849,7 +894,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -865,7 +911,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -1038,7 +1085,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -1067,7 +1115,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -1084,7 +1133,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign List", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List", "title": "Campaign List", "description": "", "lastModified": { @@ -1119,6 +1168,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1131,7 +1181,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -1147,7 +1198,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -1606,7 +1658,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -1635,7 +1688,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -1687,6 +1741,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1699,7 +1754,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -1715,7 +1771,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -2226,7 +2283,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -2255,7 +2313,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -2272,7 +2331,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile - Sent by Campaign", + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign", "title": "Mobile - Sent by Campaign", "description": "", "lastModified": { @@ -2307,6 +2366,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2319,7 +2379,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -2335,7 +2396,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -2742,7 +2804,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -2771,7 +2834,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -2823,6 +2887,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2835,7 +2900,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -2851,7 +2917,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -2894,7 +2961,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -2923,7 +2991,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -2975,6 +3044,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2987,7 +3057,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -3003,7 +3074,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -3214,7 +3286,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -3243,7 +3316,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -3295,6 +3369,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3316,7 +3391,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -3332,7 +3408,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -3418,7 +3495,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -3447,7 +3525,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -3464,7 +3543,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Requests", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests", "title": "Opened Requests", "description": "", "lastModified": { @@ -3502,6 +3581,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3514,7 +3594,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -3530,7 +3611,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -3926,7 +4008,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -3955,7 +4038,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -3972,7 +4056,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top 10 Items by Requests and YoY Change", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change", "title": "Top 10 Items by Requests and YoY Change", "description": "", "lastModified": { @@ -4010,6 +4094,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4022,7 +4107,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -4038,7 +4124,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -4402,7 +4489,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -4431,7 +4519,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -4448,7 +4537,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems", "title": "Opened Problems", "description": "", "lastModified": { @@ -4486,6 +4575,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4498,7 +4588,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -4514,7 +4605,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -4910,7 +5002,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -4939,7 +5032,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -4991,6 +5085,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5003,7 +5098,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -5019,7 +5115,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -5299,7 +5396,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -5328,7 +5426,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -5345,7 +5444,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High and Critical Priority Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems", "title": "High and Critical Priority Problems", "description": "", "lastModified": { @@ -5383,6 +5482,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5395,7 +5495,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -5411,7 +5512,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -5668,7 +5770,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -5697,7 +5800,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -5714,7 +5818,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total Incidents by Category and YoY Change", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change", "title": "Total Incidents by Category and YoY Change", "description": "", "lastModified": { @@ -5749,6 +5853,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5761,7 +5866,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -5777,7 +5883,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -6060,7 +6167,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -6089,7 +6197,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -6106,7 +6215,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known Errors", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors", "title": "Known Errors", "description": "", "lastModified": { @@ -6144,6 +6253,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6156,7 +6266,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -6172,7 +6283,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -6507,7 +6619,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -6536,7 +6649,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -6553,7 +6667,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Requests", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests", "title": "Overdue Requests", "description": "", "lastModified": { @@ -6591,6 +6705,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6603,7 +6718,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -6619,7 +6735,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -6928,7 +7045,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -6957,7 +7075,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -6974,7 +7093,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG Time to Solve an Incident", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident", "title": "AVG Time to Solve an Incident", "description": "", "lastModified": { @@ -7009,6 +7128,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7021,7 +7141,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -7037,7 +7158,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -7265,7 +7387,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -7294,7 +7417,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -7311,7 +7435,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made SLA?", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F", "title": "Made SLA?", "description": "", "lastModified": { @@ -7349,6 +7473,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7361,7 +7486,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -7377,7 +7503,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -7634,7 +7761,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -7663,7 +7791,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -7715,6 +7844,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7727,7 +7857,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -7743,7 +7874,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -7893,7 +8025,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -7922,7 +8055,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -7939,7 +8073,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems", "title": "Overdue Problems", "description": "", "lastModified": { @@ -7977,6 +8111,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7989,7 +8124,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -8005,7 +8141,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -8340,7 +8477,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -8369,7 +8507,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -8424,6 +8563,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8436,7 +8576,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -8452,7 +8593,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -8683,7 +8825,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -8712,7 +8855,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -8767,6 +8911,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8779,7 +8924,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -8795,7 +8941,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -9052,7 +9199,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -9081,7 +9229,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -9098,7 +9247,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age of Active Problems", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems", "title": "Age of Active Problems", "description": "", "lastModified": { @@ -9136,6 +9285,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9148,7 +9298,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -9164,7 +9315,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -9447,7 +9599,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -9476,7 +9629,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -9493,7 +9647,7 @@ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": {}, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Incidents", + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents", "title": "Opened Incidents", "description": "", "lastModified": { @@ -9528,6 +9682,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9540,7 +9695,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -9556,7 +9712,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -9894,7 +10051,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -9923,7 +10081,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -9975,6 +10134,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9987,7 +10147,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10003,7 +10164,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10089,7 +10251,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10118,7 +10281,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10139,7 +10303,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10195,6 +10360,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10207,7 +10373,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10223,7 +10390,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10252,7 +10420,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10303,6 +10472,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10315,7 +10485,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10331,7 +10502,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10360,7 +10532,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10409,6 +10582,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10421,7 +10595,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10437,7 +10612,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10466,7 +10642,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10529,6 +10706,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10541,7 +10719,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10557,7 +10736,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10586,7 +10766,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10930,7 +11111,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -10959,6 +11141,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -12722,7 +12905,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -12740,7 +12924,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -12756,7 +12941,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -12785,7 +12971,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -12801,7 +12988,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", "type": "TRANSFORMED" }, { @@ -12809,7 +12996,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", "type": "TRANSFORMED" } ], @@ -12886,7 +13073,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -12915,6 +13103,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -13130,7 +13319,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -13148,7 +13338,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -13164,7 +13355,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -13193,7 +13385,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -13226,7 +13419,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -13255,6 +13449,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -13642,7 +13837,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -13660,7 +13856,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -13676,7 +13873,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -13705,7 +13903,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -14140,7 +14339,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -14169,6 +14369,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -14891,7 +15092,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -14909,7 +15111,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -14925,7 +15128,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -14954,7 +15158,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -15100,7 +15305,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -15129,6 +15335,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -21719,7 +21926,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -21737,7 +21945,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -21753,7 +21962,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -21782,7 +21992,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -21971,7 +22182,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -22000,6 +22212,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -25876,7 +26089,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -25894,7 +26108,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -25910,7 +26125,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -25939,7 +26155,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -26090,7 +26307,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -26119,6 +26337,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31257,7 +31476,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -31275,7 +31495,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -31291,7 +31512,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -31320,7 +31542,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -31443,7 +31666,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -31472,6 +31696,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31674,7 +31899,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -31692,7 +31918,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -31708,7 +31935,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -31737,7 +31965,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -31786,7 +32015,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -31815,6 +32045,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -31907,7 +32138,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -31925,7 +32157,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -31941,7 +32174,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -31970,7 +32204,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -32074,7 +32309,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -32112,6 +32348,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -32360,7 +32597,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -32378,7 +32616,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -32394,7 +32633,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -32419,7 +32659,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -32553,7 +32794,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -32582,6 +32824,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -33434,7 +33677,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -33452,7 +33696,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -33479,7 +33724,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -33495,7 +33741,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -33528,7 +33775,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -33664,7 +33912,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -33683,7 +33932,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -33712,7 +33962,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -33728,7 +33979,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -33769,7 +34021,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -33917,7 +34170,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -33936,7 +34190,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -33961,7 +34216,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -34049,7 +34305,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -34068,7 +34325,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -34098,7 +34356,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -34262,7 +34521,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -34295,7 +34555,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -34531,7 +34792,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -34564,7 +34826,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -34776,7 +35039,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -34809,7 +35073,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -34973,7 +35238,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -35006,7 +35272,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -35158,7 +35425,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -35191,7 +35459,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -35331,7 +35600,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -35364,7 +35634,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -35456,7 +35727,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -35489,7 +35761,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -35557,7 +35830,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -35587,7 +35861,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -35655,7 +35930,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -35685,7 +35961,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -35981,7 +36258,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -36011,7 +36289,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -36801,7 +37080,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -36834,7 +37114,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -37718,7 +37999,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -37751,7 +38033,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -38707,7 +38990,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -38740,7 +39024,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -39564,7 +39849,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -39597,7 +39883,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -39905,7 +40192,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -39938,7 +40226,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -40786,7 +41075,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -40819,7 +41109,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -41811,7 +42102,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -41844,7 +42136,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -42704,7 +42997,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -42737,7 +43031,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -42759,7 +43054,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -42789,7 +43085,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -42811,7 +43108,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -42841,7 +43139,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -42862,7 +43161,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -42892,7 +43192,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -42908,7 +43209,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -42924,7 +43226,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -42940,7 +43243,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -42956,7 +43260,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -42972,7 +43277,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -42988,7 +43294,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43004,7 +43311,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43020,7 +43328,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43036,7 +43345,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43052,7 +43362,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43068,7 +43379,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43084,7 +43396,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43100,7 +43413,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43116,7 +43430,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43132,7 +43447,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43148,7 +43464,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43164,7 +43481,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43180,7 +43498,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43196,7 +43515,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43212,7 +43532,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43228,7 +43549,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43244,7 +43566,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43260,7 +43583,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43276,7 +43600,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43292,7 +43617,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43308,7 +43634,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43324,7 +43651,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43340,7 +43668,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43356,7 +43685,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43372,7 +43702,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43388,7 +43719,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43404,7 +43736,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43420,7 +43753,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43436,7 +43770,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43452,7 +43787,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43468,7 +43804,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43484,7 +43821,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43500,7 +43838,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43516,7 +43855,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43532,7 +43872,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43548,7 +43889,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43564,7 +43906,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43580,7 +43923,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43596,7 +43940,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43612,7 +43957,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43628,7 +43974,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43644,7 +43991,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43660,7 +44008,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43676,7 +44025,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43692,7 +44042,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43708,7 +44059,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43724,7 +44076,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43740,7 +44093,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43756,7 +44110,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43772,7 +44127,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43788,7 +44144,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43804,7 +44161,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43820,7 +44178,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43836,7 +44195,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43852,7 +44212,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43868,7 +44229,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43884,7 +44246,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43900,7 +44263,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43916,7 +44280,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43932,7 +44297,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43948,7 +44314,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43964,7 +44331,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43980,7 +44348,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -43996,7 +44365,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -44012,7 +44382,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -44028,7 +44399,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -44044,7 +44416,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -44060,7 +44433,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -44076,7 +44450,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -44092,7 +44467,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -44108,7 +44484,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -44124,7 +44501,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } }, { @@ -44140,2033 +44518,8 @@ "systemMetadata": { "lastObserved": 1638860400000, "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:89fbc376e636670d29a60ef2347c156f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:ba8a5ac7eb4c6e5edc9b03bf8891be55", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.222d1406-de0e-cd8d-0b94-9b45a0007e59)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5", - "urn": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.38130558-4194-2e2a-3046-c0d887829cb4)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5", - "urn": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.692a2da4-2a82-32c1-f713-63b8e4325d86)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5", - "urn": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5", - "urn": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.8a6a269a-d6de-fae4-5050-513255b40ffc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", - "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.c57a5574-db47-46df-677f-0b708dab14db)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", - "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.e604255e-0573-3951-6db7-05bee48116c1)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", - "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.20fc5eb7-81eb-aa18-8c39-af501c62d085)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.373c6466-bb0c-b319-8752-632456349261)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.58af9ecf-b839-da50-65e1-2e1fa20e3362)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.618b3e76-75c1-cb31-0c61-3f4890b72c31)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.721c3c41-7a2b-16a8-3281-6f948a44be96)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.7ef184c1-5a41-5ec8-723e-ae44c20aa335)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.7fbc77ba-0ab6-3727-0db3-d8402a804da5)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.8385ea9a-0749-754f-7ad9-824433de2120)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.b207c2f2-b675-32e3-2663-17bb836a018b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.c14973c2-e1c3-563a-a9c1-8a408396d22a)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.f76d3570-23b8-f74b-d85c-cc5484c2079c)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.130496dc-29ca-8a89-e32b-d73c4d8b65ff)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:ba8a5ac7eb4c6e5edc9b03bf8891be55", - "urn": "urn:li:container:ba8a5ac7eb4c6e5edc9b03bf8891be55" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,acryl_site1.8f7dd564-36b6-593f-3c6f-687ad06cd40b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5", - "urn": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,acryl_site1.20e44c22-1ccd-301a-220c-7b6837d09a52)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", - "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,acryl_site1.39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", - "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,acryl_site1.5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5", - "urn": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", - "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", - "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", - "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", - "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:ba8a5ac7eb4c6e5edc9b03bf8891be55", - "urn": "urn:li:container:ba8a5ac7eb4c6e5edc9b03bf8891be55" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.5449c627-7462-4ef7-b492-bda46be068e3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:ba8a5ac7eb4c6e5edc9b03bf8891be55", - "urn": "urn:li:container:ba8a5ac7eb4c6e5edc9b03bf8891be55" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "Samples" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - }, - { - "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", - "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", - "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "SubProject1" - }, - { - "id": "AbcJoinWorkbook" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Email Performance by Campaign" - }, - { - "id": "Marketo" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Dvdrental Workbook" - }, - { - "id": "actor+ (dvdrental)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Dvdrental Workbook" - }, - { - "id": "actor+ (dvdrental)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "Samples" - }, - { - "id": "Superstore Datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Requests" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Problems" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Incidents" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Executive Dashboard" - }, - { - "id": "Incidents" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Customer Payment Query" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "Customer Payment Query" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - }, - { - "id": "acryl_site1" - }, - { - "id": "default" - }, - { - "id": "test publish datasource" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_with_platform_instance" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py index 57fcb0b6ee49a..b64609b6ea605 100644 --- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py +++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py @@ -8,9 +8,11 @@ import pytest from freezegun import freeze_time from requests.adapters import ConnectionError +from tableauserverclient import Server from tableauserverclient.models import ( DatasourceItem, ProjectItem, + SiteItem, ViewItem, WorkbookItem, ) @@ -18,7 +20,12 @@ from datahub.configuration.source_common import DEFAULT_ENV from datahub.emitter.mce_builder import make_schema_field_urn from datahub.ingestion.run.pipeline import Pipeline, PipelineContext -from datahub.ingestion.source.tableau import TableauConfig, TableauSource +from datahub.ingestion.source.tableau import ( + TableauConfig, + TableauSiteSource, + TableauSource, + TableauSourceReport, +) from datahub.ingestion.source.tableau_common import ( TableauLineageOverrides, TableauUpstreamReference, @@ -124,6 +131,25 @@ def side_effect_project_data(*arg, **kwargs): return [project1, project2, project3, project4], mock_pagination +def side_effect_site_data(*arg, **kwargs): + mock_pagination = mock.MagicMock() + mock_pagination.total_available = None + + site1: SiteItem = SiteItem(name="Acryl", content_url="acryl") + site1._id = "190a6a5c-63ed-4de1-8045-site1" + site1.state = "Active" + + site2: SiteItem = SiteItem(name="Site 2", content_url="site2") + site2._id = "190a6a5c-63ed-4de1-8045-site2" + site2.state = "Active" + + site3: SiteItem = SiteItem(name="Site 3", content_url="site3") + site3._id = "190a6a5c-63ed-4de1-8045-site3" + site3.state = "Suspended" + + return [site1, site2, site3], mock_pagination + + def side_effect_datasource_data(*arg, **kwargs): mock_pagination = mock.MagicMock() mock_pagination.total_available = None @@ -199,6 +225,13 @@ def side_effect_datasource_get_by_id(id, *arg, **kwargs): return ds +def side_effect_site_get_by_id(id, *arg, **kwargs): + sites, _ = side_effect_site_data() + for site in sites: + if site._id == id: + return site + + def tableau_ingest_common( pytestconfig, tmp_path, @@ -223,9 +256,14 @@ def tableau_ingest_common( mocked_metadata.query.side_effect = side_effect_query_metadata_response mock_client.metadata = mocked_metadata mock_client.auth = mock.Mock() + mock_client.site_id = "190a6a5c-63ed-4de1-8045-site1" mock_client.views = mock.Mock() mock_client.projects = mock.Mock() + mock_client.sites = mock.Mock() + mock_client.projects.get.side_effect = side_effect_project_data + mock_client.sites.get.side_effect = side_effect_site_data + mock_client.sites.get_by_id.side_effect = side_effect_site_get_by_id mock_client.datasources = mock.Mock() mock_client.datasources.get.side_effect = datasources_side_effect mock_client.datasources.get_by_id.side_effect = ( @@ -375,7 +413,7 @@ def test_project_pattern(pytestconfig, tmp_path, mock_datahub_graph): output_file_name, mock_datahub_graph, pipeline_config=new_config, - pipeline_name="test_project_pattern", + pipeline_name="test_tableau_ingest", ) @@ -676,6 +714,7 @@ def test_tableau_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph) golden_file_name, output_file_name, mock_datahub_graph, + pipeline_name="test_tableau_ingest", ) checkpoint1 = get_current_checkpoint_from_pipeline(pipeline_run1) @@ -689,6 +728,7 @@ def test_tableau_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph) golden_file_deleted_name, output_file_deleted_name, mock_datahub_graph, + pipeline_name="test_tableau_ingest", ) checkpoint2 = get_current_checkpoint_from_pipeline(pipeline_run2) @@ -891,9 +931,16 @@ def test_lineage_metadata( "user_source": "user_source", } - source = TableauSource(config=config, ctx=context) + site_source = TableauSiteSource( + config=config, + ctx=context, + platform="tableau", + site=SiteItem(name="Site 1", content_url="site1"), + report=TableauSourceReport(), + server=Server("https://test-tableau-server.com"), + ) - lineage = source._create_lineage_from_unsupported_csql( + lineage = site_source._create_lineage_from_unsupported_csql( csql_urn=csql_urn, csql={ "query": "SELECT user_id, source, user_source FROM (SELECT *, ROW_NUMBER() OVER (partition BY user_id ORDER BY __partition_day DESC) AS rank_ FROM invent_dw.UserDetail ) source_user WHERE rank_ = 1", @@ -914,7 +961,7 @@ def test_lineage_metadata( ) # With database as None - lineage = source._create_lineage_from_unsupported_csql( + lineage = site_source._create_lineage_from_unsupported_csql( csql_urn=csql_urn, csql={ "query": "SELECT user_id, source, user_source FROM (SELECT *, ROW_NUMBER() OVER (partition BY user_id ORDER BY __partition_day DESC) AS rank_ FROM my_bigquery_project.invent_dw.UserDetail ) source_user WHERE rank_ = 1", @@ -955,3 +1002,107 @@ def test_get_all_datasources_failure(pytestconfig, tmp_path, mock_datahub_graph) pipeline_name="test_tableau_ingest", datasources_side_effect=ValueError("project_id must be defined."), ) + + +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_tableau_ingest_multiple_sites(pytestconfig, tmp_path, mock_datahub_graph): + enable_logging() + output_file_name: str = "tableau_mces_multiple_sites.json" + golden_file_name: str = "tableau_multiple_sites_mces_golden.json" + + new_pipeline_config: Dict[Any, Any] = { + **config_source_default, + "add_site_container": True, + "ingest_multiple_sites": True, + } + + tableau_ingest_common( + pytestconfig=pytestconfig, + tmp_path=tmp_path, + side_effect_query_metadata_response=[ + read_response(pytestconfig, "workbooksConnection_all.json"), + read_response(pytestconfig, "sheetsConnection_all.json"), + read_response(pytestconfig, "dashboardsConnection_all.json"), + read_response(pytestconfig, "embeddedDatasourcesConnection_all.json"), + read_response(pytestconfig, "publishedDatasourcesConnection_all.json"), + read_response(pytestconfig, "customSQLTablesConnection_all.json"), + read_response(pytestconfig, "databaseTablesConnection_all.json"), + read_response(pytestconfig, "workbooksConnection_all.json"), + read_response(pytestconfig, "sheetsConnection_all.json"), + read_response(pytestconfig, "dashboardsConnection_all.json"), + read_response(pytestconfig, "embeddedDatasourcesConnection_all.json"), + read_response(pytestconfig, "publishedDatasourcesConnection_all.json"), + read_response(pytestconfig, "customSQLTablesConnection_all.json"), + read_response(pytestconfig, "databaseTablesConnection_all.json"), + ], + golden_file_name=golden_file_name, + output_file_name=output_file_name, + mock_datahub_graph=mock_datahub_graph, + pipeline_name="test_tableau_multiple_site_ingestion", + pipeline_config=new_pipeline_config, + ) + + +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_tableau_ingest_sites_as_container(pytestconfig, tmp_path, mock_datahub_graph): + enable_logging() + output_file_name: str = "tableau_mces_ingest_sites_as_container.json" + golden_file_name: str = "tableau_sites_as_container_mces_golden.json" + + new_pipeline_config: Dict[Any, Any] = { + **config_source_default, + "add_site_container": True, + } + + tableau_ingest_common( + pytestconfig=pytestconfig, + tmp_path=tmp_path, + side_effect_query_metadata_response=[ + read_response(pytestconfig, "workbooksConnection_all.json"), + read_response(pytestconfig, "sheetsConnection_all.json"), + read_response(pytestconfig, "dashboardsConnection_all.json"), + read_response(pytestconfig, "embeddedDatasourcesConnection_all.json"), + read_response(pytestconfig, "publishedDatasourcesConnection_all.json"), + read_response(pytestconfig, "customSQLTablesConnection_all.json"), + read_response(pytestconfig, "databaseTablesConnection_all.json"), + ], + golden_file_name=golden_file_name, + output_file_name=output_file_name, + mock_datahub_graph=mock_datahub_graph, + pipeline_name="test_tableau_multiple_site_ingestion", + pipeline_config=new_pipeline_config, + ) + + +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_site_name_pattern(pytestconfig, tmp_path, mock_datahub_graph): + enable_logging() + output_file_name: str = "tableau_site_name_pattern_mces.json" + golden_file_name: str = "tableau_site_name_pattern_mces_golden.json" + + new_config = config_source_default.copy() + new_config["ingest_multiple_sites"] = True + new_config["add_site_container"] = True + new_config["site_name_pattern"] = {"allow": ["^Site.*$"]} + + tableau_ingest_common( + pytestconfig, + tmp_path, + [ + read_response(pytestconfig, "workbooksConnection_all.json"), + read_response(pytestconfig, "sheetsConnection_all.json"), + read_response(pytestconfig, "dashboardsConnection_all.json"), + read_response(pytestconfig, "embeddedDatasourcesConnection_all.json"), + read_response(pytestconfig, "publishedDatasourcesConnection_all.json"), + read_response(pytestconfig, "customSQLTablesConnection_all.json"), + read_response(pytestconfig, "databaseTablesConnection_all.json"), + ], + golden_file_name, + output_file_name, + mock_datahub_graph, + pipeline_config=new_config, + pipeline_name="test_tableau_site_name_pattern_ingest", + ) diff --git a/metadata-ingestion/tests/integration/trino/docker-compose.yml b/metadata-ingestion/tests/integration/trino/docker-compose.yml index 59ba91e46d543..5982e6af2d335 100644 --- a/metadata-ingestion/tests/integration/trino/docker-compose.yml +++ b/metadata-ingestion/tests/integration/trino/docker-compose.yml @@ -1,9 +1,6 @@ # Adapted from https://github.com/big-data-europe/docker-hive. -version: "3" - services: - testtrino: image: trinodb/trino:369 container_name: "testtrino" diff --git a/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json index ef535158165da..cd1cd0d7e28a4 100644 --- a/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json @@ -9,11 +9,7 @@ "customProperties": { "platform": "vertica", "env": "PROD", - "database": "vmart", - "cluster_type": "Enterprise", - "cluster_size": "101 GB", - "subcluster": " ", - "communal_storage_path": "" + "database": "vmart" }, "name": "vmart" } @@ -101,10 +97,7 @@ "platform": "vertica", "env": "PROD", "database": "vmart", - "schema": "public", - "projection_count": "12", - "udx_list": "APPROXIMATE_COUNT_DISTINCT_SYNOPSIS_INFO, APPROXIMATE_MEDIAN, APPROXIMATE_PERCENTILE, AcdDataToCount, AcdDataToLongSyn, AcdDataToSyn, AcdSynToCount, AcdSynToSyn, DelimitedExport, DelimitedExportMulti, EmptyMap, Explode, FAvroParser, FCefParser, FCsvParser, FDelimitedPairParser, FDelimitedParser, FIDXParser, FJSONParser, FRegexParser, FlexTokenizer, JsonExport, JsonExportMulti, KafkaAvroParser, KafkaCheckBrokers, KafkaExport, KafkaInsertDelimiters, KafkaInsertLengths, KafkaJsonParser, KafkaListManyTopics, KafkaListTopics, KafkaOffsets, KafkaParser, KafkaSource, KafkaTopicDetails, MSE, MapAggregate, MapAggregate, MapContainsKey, MapContainsKey, MapContainsValue, MapContainsValue, MapDelimitedExtractor, MapItems, MapItems, MapJSONExtractor, MapKeys, MapKeys, MapKeysInfo, MapKeysInfo, MapLookup, MapLookup, MapLookup, MapPut, MapRegexExtractor, MapSize, MapSize, MapToString, MapToString, MapValues, MapValues, MapValuesOrField, MapVersion, MapVersion, OrcExport, OrcExportMulti, PRC, ParquetExport, ParquetExportMulti, PickBestType, PickBestType, PickBestType, ROC, STV_AsGeoJSON, STV_AsGeoJSON, STV_AsGeoJSON, STV_Create_Index, STV_Create_Index, STV_Create_Index, STV_DWithin, STV_DWithin, STV_DWithin, STV_Describe_Index, STV_Drop_Index, STV_Export2Shapefile, STV_Extent, STV_Extent, STV_ForceLHR, STV_Geography, STV_Geography, STV_GeographyPoint, STV_Geometry, STV_Geometry, STV_GeometryPoint, STV_GeometryPoint, STV_GetExportShapefileDirectory, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_IsValidReason, STV_IsValidReason, STV_IsValidReason, STV_LineStringPoint, STV_LineStringPoint, STV_LineStringPoint, STV_MemSize, STV_MemSize, STV_MemSize, STV_NN, STV_NN, STV_NN, STV_PolygonPoint, STV_PolygonPoint, STV_PolygonPoint, STV_Refresh_Index, STV_Refresh_Index, STV_Refresh_Index, STV_Rename_Index, STV_Reverse, STV_SetExportShapefileDirectory, STV_ShpCreateTable, STV_ShpParser, STV_ShpSource, ST_Area, ST_Area, ST_Area, ST_AsBinary, ST_AsBinary, ST_AsBinary, ST_AsText, ST_AsText, ST_AsText, ST_Boundary, ST_Buffer, ST_Centroid, ST_Contains, ST_Contains, ST_Contains, ST_ConvexHull, ST_Crosses, ST_Difference, ST_Disjoint, ST_Disjoint, ST_Disjoint, ST_Distance, ST_Distance, ST_Distance, ST_Envelope, ST_Equals, ST_Equals, ST_Equals, ST_GeoHash, ST_GeoHash, ST_GeoHash, ST_GeographyFromText, ST_GeographyFromWKB, ST_GeomFromGeoHash, ST_GeomFromGeoJSON, ST_GeomFromGeoJSON, ST_GeomFromText, ST_GeomFromText, ST_GeomFromWKB, ST_GeomFromWKB, ST_GeometryN, ST_GeometryN, ST_GeometryN, ST_GeometryType, ST_GeometryType, ST_GeometryType, ST_Intersection, ST_Intersects, ST_Intersects, ST_IsEmpty, ST_IsEmpty, ST_IsEmpty, ST_IsSimple, ST_IsSimple, ST_IsSimple, ST_IsValid, ST_IsValid, ST_IsValid, ST_Length, ST_Length, ST_Length, ST_NumGeometries, ST_NumGeometries, ST_NumGeometries, ST_NumPoints, ST_NumPoints, ST_NumPoints, ST_Overlaps, ST_PointFromGeoHash, ST_PointN, ST_PointN, ST_PointN, ST_Relate, ST_SRID, ST_SRID, ST_SRID, ST_Simplify, ST_SimplifyPreserveTopology, ST_SymDifference, ST_Touches, ST_Touches, ST_Touches, ST_Transform, ST_Union, ST_Union, ST_Within, ST_Within, ST_Within, ST_X, ST_X, ST_X, ST_XMax, ST_XMax, ST_XMax, ST_XMin, ST_XMin, ST_XMin, ST_Y, ST_Y, ST_Y, ST_YMax, ST_YMax, ST_YMax, ST_YMin, ST_YMin, ST_YMin, ST_intersects, SetMapKeys, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_NumCol, Unnest, VoltageSecureAccess, VoltageSecureAccess, VoltageSecureConfigure, VoltageSecureConfigureGlobal, VoltageSecureProtect, VoltageSecureProtect, VoltageSecureProtectAllKeys, VoltageSecureRefreshPolicy, VoltageSecureVersion, append_centers, apply_bisecting_kmeans, apply_iforest, apply_inverse_pca, apply_inverse_svd, apply_kmeans, apply_kprototypes, apply_normalize, apply_one_hot_encoder, apply_pca, apply_svd, approximate_quantiles, ar_create_blobs, ar_final_newton, ar_save_model, ar_transition_newton, arima_bfgs, arima_line_search, arima_save_model, avg_all_columns_local, bisecting_kmeans_init_model, bk_apply_best_kmeans_results, bk_compute_totss_local, bk_finalize_model, bk_get_rows_in_active_cluster, bk_kmeans_compute_local_centers, bk_kmeans_compute_withinss, bk_kmeans_fast_random_init, bk_kmeans_slow_random_init, bk_kmeanspp_init_cur_cluster, bk_kmeanspp_reset_blob, bk_kmeanspp_select_new_centers, bk_kmeanspp_within_chunk_sum, bk_save_final_model, bk_write_new_cluster_level, blob_to_table, bufUdx, bufUdx, calc_pseudo_centers, calculate_alpha_linear, calculate_hessian_linear1, calculate_hessian_linear2, chi_squared, cleanup_kmeans_files, compute_and_save_global_center, compute_and_save_new_centers, compute_local_totss, compute_local_withinss, compute_new_local_centers, confusion_matrix, coordinate_descent_covariance, corr_matrix, count_rows_in_blob, create_aggregator_blob, error_rate, evaluate_naive_bayes_model, evaluate_reg_model, evaluate_svm_model, export_model_files, finalize_blob_resource_group, get_attr_minmax, get_attr_robust_zscore, get_attr_zscore, get_model_attribute, get_model_summary, get_robust_zscore_median, iforest_create_blobs, iforest_phase0_udf1, iforest_phase0_udf2, iforest_phase1_udf1, iforest_phase1_udf2, iforest_phase1_udf3, iforest_phase1_udf4, iforest_phase2_udf1, iforest_phase2_udf2, iforest_phase2_udf3, iforest_phase2_udf4, iforest_save_model, import_model_files, isOrContains, kmeansAddMetricsToModel, kmeans_init_blobs, kmeans_to_write_final_centers, lift_table, line_search_logistic1, line_search_logistic2, load_rows_into_blocks, map_factor, math_op, matrix_global_xtx, matrix_local_xtx, mode_finder, model_converter, naive_bayes_phase1, naive_bayes_phase1_blob, naive_bayes_phase2, pca_prep1_global, pca_prep1_local, pca_prep2, pmml_parser, predict_arima, predict_autoregressor, predict_linear_reg, predict_logistic_reg, predict_moving_average, predict_naive_bayes, predict_naive_bayes_classes, predict_pmml, predict_poisson_reg, predict_rf_classifier, predict_rf_classifier_classes, predict_rf_regressor, predict_svm_classifier, predict_svm_regressor, predict_xgb_classifier, predict_xgb_classifier_classes, predict_xgb_regressor, random_init, random_init_write, read_from_dfblob, read_map_factor, read_ptree, read_tree, reg_final_bfgs, reg_final_newton, reg_transition_bfgs, reg_transition_newton, reg_write_model, remove_blob, reverse_normalize, rf_blob, rf_clean, rf_phase0_udf1, rf_phase0_udf2, rf_phase1_udf1, rf_phase1_udf2, rf_phase1_udf3, rf_phase1_udf4, rf_phase2_udf1, rf_phase2_udf2, rf_phase2_udf3, rf_phase2_udf4, rf_predictor_importance, rf_save_model, rsquared, save_cv_result, save_pca_model, save_svd_model, save_svm_model, select_new_centers, store_minmax_model, store_one_hot_encoder_model, store_robust_zscore_model, store_zscore_model, table_to_blob, table_to_dfblob, tokenize, topk, update_and_return_sum_of_squared_distances, upgrade_model_format, writeInitialKmeansModelToDfs, xgb_create_blobs, xgb_phase0_udf1, xgb_phase0_udf2, xgb_phase1_udf1, xgb_phase1_udf2, xgb_phase1_udf3, xgb_phase2_udf1, xgb_phase2_udf2, xgb_phase2_udf3, xgb_predictor_importance, xgb_prune, xgb_save_model, yule_walker, ", - "udx_language": "ComplexTypesLib -- Functions for Complex Types | DelimitedExportLib -- Delimited data export package | JsonExportLib -- Json data export package | MachineLearningLib -- Machine learning package | OrcExportLib -- Orc export package | ParquetExportLib -- Parquet export package | ApproximateLib -- Approximate package | FlexTableLib -- Flexible Tables Data Load and Query | KafkaLib -- Kafka streaming load and export | PlaceLib -- Geospatial package | VoltageSecureLib -- Voltage SecureData Connector | TransformFunctions -- User-defined Python library | " + "schema": "public" }, "name": "public" } @@ -204,7 +197,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -215,6 +208,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -229,7 +223,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -246,7 +240,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -256,7 +250,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:23:05.308022+00:00", + "create_time": "2024-06-03 12:24:31.057395+00:00", "table_size": "0 KB" }, "name": "clicks", @@ -266,7 +260,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.clicks", + "schemaName": "Vmart.public.clicks", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -337,7 +331,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -355,7 +349,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -380,7 +374,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -391,6 +385,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -405,7 +400,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -422,7 +417,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -432,7 +427,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.846965+00:00", + "create_time": "2024-06-03 12:23:45.887434+00:00", "table_size": "2119 KB" }, "name": "customer_dimension", @@ -442,7 +437,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.customer_dimension", + "schemaName": "Vmart.public.customer_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -747,7 +742,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -765,7 +760,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -790,7 +785,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -801,6 +796,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -815,7 +811,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -832,7 +828,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -842,8 +838,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.857152+00:00", - "table_size": "138 KB" + "create_time": "2024-06-03 12:23:45.897002+00:00", + "table_size": "145 KB" }, "name": "date_dimension", "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", @@ -852,7 +848,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.date_dimension", + "schemaName": "Vmart.public.date_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -1157,7 +1153,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1175,7 +1171,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1200,7 +1196,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -1211,6 +1207,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1225,7 +1222,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -1242,7 +1239,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -1252,7 +1249,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.863745+00:00", + "create_time": "2024-06-03 12:23:45.903227+00:00", "table_size": "327 KB" }, "name": "employee_dimension", @@ -1262,7 +1259,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.employee_dimension", + "schemaName": "Vmart.public.employee_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -1528,7 +1525,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1546,7 +1543,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1571,7 +1568,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -1582,6 +1579,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1596,7 +1594,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -1613,7 +1611,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -1623,8 +1621,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.873181+00:00", - "table_size": "2564 KB" + "create_time": "2024-06-03 12:23:45.912348+00:00", + "table_size": "2567 KB" }, "name": "inventory_fact", "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", @@ -1633,7 +1631,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.inventory_fact", + "schemaName": "Vmart.public.inventory_fact", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -1743,7 +1741,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1761,7 +1759,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1786,7 +1784,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -1797,6 +1795,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1811,7 +1810,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -1828,7 +1827,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -1838,7 +1837,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:23:05.408507+00:00", + "create_time": "2024-06-03 12:24:31.194163+00:00", "table_size": "0 KB" }, "name": "phrases", @@ -1848,7 +1847,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.phrases", + "schemaName": "Vmart.public.phrases", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -1893,7 +1892,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1911,7 +1910,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1936,7 +1935,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -1947,6 +1946,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1961,7 +1961,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -1978,7 +1978,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -1988,7 +1988,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.850505+00:00", + "create_time": "2024-06-03 12:23:45.890782+00:00", "table_size": "19 KB" }, "name": "product_dimension", @@ -1998,7 +1998,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.product_dimension", + "schemaName": "Vmart.public.product_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -2303,7 +2303,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2321,7 +2321,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -2346,7 +2346,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -2357,6 +2357,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2371,7 +2372,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -2388,7 +2389,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -2398,7 +2399,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.853878+00:00", + "create_time": "2024-06-03 12:23:45.893891+00:00", "table_size": "3 KB" }, "name": "promotion_dimension", @@ -2408,7 +2409,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.promotion_dimension", + "schemaName": "Vmart.public.promotion_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -2596,7 +2597,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2614,7 +2615,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -2639,7 +2640,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -2650,6 +2651,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2664,7 +2666,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -2681,7 +2683,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -2691,7 +2693,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:23:05.296044+00:00", + "create_time": "2024-06-03 12:24:31.046829+00:00", "table_size": "0 KB" }, "name": "readings", @@ -2701,7 +2703,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.readings", + "schemaName": "Vmart.public.readings", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -2772,7 +2774,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2790,7 +2792,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -2815,7 +2817,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -2826,6 +2828,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -2840,7 +2843,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -2857,7 +2860,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -2867,7 +2870,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.867119+00:00", + "create_time": "2024-06-03 12:23:45.906471+00:00", "table_size": "1 KB" }, "name": "shipping_dimension", @@ -2877,7 +2880,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.shipping_dimension", + "schemaName": "Vmart.public.shipping_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -2961,7 +2964,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2979,7 +2982,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -3004,7 +3007,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -3015,6 +3018,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3029,7 +3033,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -3046,7 +3050,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -3056,7 +3060,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.860541+00:00", + "create_time": "2024-06-03 12:23:45.900185+00:00", "table_size": "1 KB" }, "name": "vendor_dimension", @@ -3066,7 +3070,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.vendor_dimension", + "schemaName": "Vmart.public.vendor_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -3202,7 +3206,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3220,7 +3224,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -3245,7 +3249,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vmart_load_success,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vmart_load_success,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -3256,6 +3260,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3270,7 +3275,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vmart_load_success,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vmart_load_success,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -3287,7 +3292,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vmart_load_success,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vmart_load_success,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -3297,7 +3302,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:23:04.970568+00:00", + "create_time": "2024-06-03 12:24:30.233405+00:00", "table_size": "0 KB" }, "name": "vmart_load_success", @@ -3307,7 +3312,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.vmart_load_success", + "schemaName": "Vmart.public.vmart_load_success", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -3352,7 +3357,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vmart_load_success,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vmart_load_success,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3370,7 +3375,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vmart_load_success,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vmart_load_success,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -3395,7 +3400,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -3406,6 +3411,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3420,7 +3426,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -3437,7 +3443,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -3447,7 +3453,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.870169+00:00", + "create_time": "2024-06-03 12:23:45.909432+00:00", "table_size": "2 KB" }, "name": "warehouse_dimension", @@ -3457,7 +3463,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.warehouse_dimension", + "schemaName": "Vmart.public.warehouse_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -3567,7 +3573,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3585,7 +3591,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -3610,7 +3616,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.sampleview,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -3621,6 +3627,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3635,7 +3642,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.sampleview,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -3652,7 +3659,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.sampleview,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -3662,7 +3669,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:23:05.319029+00:00", + "create_time": "2024-06-03 12:24:31.075640+00:00", "table_size": "0 KB", "view_definition": "SELECT sum(customer_dimension.annual_income) AS SUM, customer_dimension.customer_state FROM public.customer_dimension WHERE (customer_dimension.customer_key IN (SELECT store_sales_fact.customer_key FROM store.store_sales_fact)) GROUP BY customer_dimension.customer_state ORDER BY customer_dimension.customer_state", "is_view": "True" @@ -3674,7 +3681,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.sampleview", + "schemaName": "Vmart.public.sampleview", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -3693,7 +3700,7 @@ }, "fields": [ { - "fieldPath": "SUM", + "fieldPath": "sum", "nullable": true, "description": "", "type": { @@ -3732,7 +3739,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.sampleview,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3750,7 +3757,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.sampleview,PROD)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { @@ -3768,27 +3775,19 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.sampleview,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "browsePathsV2", "aspect": { "json": { - "upstreams": [ + "path": [ { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", - "type": "TRANSFORMED" + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", - "type": "TRANSFORMED" + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -3800,8 +3799,96 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.sampleview,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "vertica", + "env": "PROD", + "database": "vmart", + "schema": "store" + }, + "name": "store" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:vertica" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:343f520ad0fb3259b298736800bb1385" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -3810,10 +3897,6 @@ { "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" - }, - { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -3826,7 +3909,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -3837,6 +3920,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -3851,12 +3935,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" } }, "systemMetadata": { @@ -3868,7 +3952,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -3878,23 +3962,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "ROS_Count": "1", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(date_dimension.date_key)", - "Projection_size": "138 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" + "create_time": "2024-06-03 12:23:45.918904+00:00", + "table_size": "2 KB" }, - "name": "date_dimension_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "name": "store_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.date_dimension_super", + "schemaName": "Vmart.store.store_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -3913,7 +3991,7 @@ }, "fields": [ { - "fieldPath": "date_key", + "fieldPath": "store_key", "nullable": true, "description": "", "type": { @@ -3923,36 +4001,10 @@ }, "nativeDataType": "INTEGER()", "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "date", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "full_date_description", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=18)", - "recursive": false, - "isPartOfKey": false + "isPartOfKey": true }, { - "fieldPath": "day_of_week", + "fieldPath": "store_name", "nullable": true, "description": "", "type": { @@ -3960,12 +4012,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=9)", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "day_number_in_calendar_month", + "fieldPath": "store_number", "nullable": true, "description": "", "type": { @@ -3978,85 +4030,85 @@ "isPartOfKey": false }, { - "fieldPath": "day_number_in_calendar_year", + "fieldPath": "store_address", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=256)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "day_number_in_fiscal_month", + "fieldPath": "store_city", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "day_number_in_fiscal_year", + "fieldPath": "store_state", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=2)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "last_day_in_week_indicator", + "fieldPath": "store_region", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "last_day_in_month_indicator", + "fieldPath": "floor_plan_type", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "calendar_week_number_in_year", + "fieldPath": "photo_processing_type", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "calendar_month_name", + "fieldPath": "financial_service_type", "nullable": true, "description": "", "type": { @@ -4064,12 +4116,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=9)", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "calendar_month_number_in_year", + "fieldPath": "selling_square_footage", "nullable": true, "description": "", "type": { @@ -4082,20 +4134,7 @@ "isPartOfKey": false }, { - "fieldPath": "calendar_year_month", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "CHAR(length=7)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "calendar_quarter", + "fieldPath": "total_square_footage", "nullable": true, "description": "", "type": { @@ -4108,33 +4147,33 @@ "isPartOfKey": false }, { - "fieldPath": "calendar_year_quarter", + "fieldPath": "first_open_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "calendar_half_year", + "fieldPath": "last_remodel_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "calendar_year", + "fieldPath": "number_of_employees", "nullable": true, "description": "", "type": { @@ -4147,41 +4186,41 @@ "isPartOfKey": false }, { - "fieldPath": "holiday_indicator", + "fieldPath": "annual_shrinkage", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=10)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "weekday_indicator", + "fieldPath": "foot_traffic", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "selling_season", + "fieldPath": "monthly_rent_cost", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false } @@ -4199,13 +4238,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Projections" + "Table" ] } }, @@ -4217,44 +4256,19 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "browsePathsV2", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ + "path": [ { "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" } ] } @@ -4267,7 +4281,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -4278,6 +4292,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4292,12 +4307,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" } }, "systemMetadata": { @@ -4309,7 +4324,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -4319,23 +4334,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "ROS_Count": "1", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(product_dimension.product_key, product_dimension.product_version)", - "Projection_size": "19 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" + "create_time": "2024-06-03 12:23:45.929154+00:00", + "table_size": "8646 KB" }, - "name": "product_dimension_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "name": "store_orders_fact", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.product_dimension_super", + "schemaName": "Vmart.store.store_orders_fact", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -4380,137 +4389,111 @@ "isPartOfKey": false }, { - "fieldPath": "product_description", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=128)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sku_number", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "CHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "category_description", + "fieldPath": "store_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "department_description", + "fieldPath": "vendor_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "package_type_description", + "fieldPath": "employee_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "package_size", + "fieldPath": "order_number", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "fat_content", + "fieldPath": "date_ordered", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "diet_type", + "fieldPath": "date_shipped", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "weight", + "fieldPath": "expected_delivery_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "weight_units_of_measure", + "fieldPath": "date_delivered", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "shelf_width", + "fieldPath": "quantity_ordered", "nullable": true, "description": "", "type": { @@ -4523,7 +4506,7 @@ "isPartOfKey": false }, { - "fieldPath": "shelf_height", + "fieldPath": "quantity_delivered", "nullable": true, "description": "", "type": { @@ -4536,20 +4519,20 @@ "isPartOfKey": false }, { - "fieldPath": "shelf_depth", + "fieldPath": "shipper_name", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "product_price", + "fieldPath": "unit_price", "nullable": true, "description": "", "type": { @@ -4562,7 +4545,7 @@ "isPartOfKey": false }, { - "fieldPath": "product_cost", + "fieldPath": "shipping_cost", "nullable": true, "description": "", "type": { @@ -4575,7 +4558,7 @@ "isPartOfKey": false }, { - "fieldPath": "lowest_competitor_price", + "fieldPath": "total_order_cost", "nullable": true, "description": "", "type": { @@ -4588,7 +4571,7 @@ "isPartOfKey": false }, { - "fieldPath": "highest_competitor_price", + "fieldPath": "quantity_in_stock", "nullable": true, "description": "", "type": { @@ -4601,7 +4584,7 @@ "isPartOfKey": false }, { - "fieldPath": "average_competitor_price", + "fieldPath": "reorder_level", "nullable": true, "description": "", "type": { @@ -4614,7 +4597,7 @@ "isPartOfKey": false }, { - "fieldPath": "discontinued_flag", + "fieldPath": "overstock_ceiling", "nullable": true, "description": "", "type": { @@ -4640,38 +4623,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Projections" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", - "type": "TRANSFORMED" - } + "Table" ] } }, @@ -4683,7 +4641,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -4694,8 +4652,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" } ] } @@ -4708,7 +4666,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -4719,6 +4677,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -4733,12 +4692,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" } }, "systemMetadata": { @@ -4750,7 +4709,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -4760,23 +4719,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "ROS_Count": "1", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(promotion_dimension.promotion_key)", - "Projection_size": "3 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" + "create_time": "2024-06-03 12:23:45.922050+00:00", + "table_size": "225096 KB" }, - "name": "promotion_dimension_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "name": "store_sales_fact", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.promotion_dimension_super", + "schemaName": "Vmart.store.store_sales_fact", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -4795,7 +4748,7 @@ }, "fields": [ { - "fieldPath": "promotion_key", + "fieldPath": "date_key", "nullable": true, "description": "", "type": { @@ -4808,111 +4761,111 @@ "isPartOfKey": false }, { - "fieldPath": "promotion_name", + "fieldPath": "product_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "price_reduction_type", + "fieldPath": "product_version", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "promotion_media_type", + "fieldPath": "store_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "ad_type", + "fieldPath": "promotion_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "display_type", + "fieldPath": "customer_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "coupon_type", + "fieldPath": "employee_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "ad_media_name", + "fieldPath": "pos_transaction_number", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "display_provider", + "fieldPath": "sales_quantity", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "promotion_cost", + "fieldPath": "sales_dollar_amount", "nullable": true, "description": "", "type": { @@ -4925,77 +4878,117 @@ "isPartOfKey": false }, { - "fieldPath": "promotion_begin_date", + "fieldPath": "cost_dollar_amount", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "promotion_end_date", + "fieldPath": "gross_profit_dollar_amount", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Projections" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" + }, + { + "fieldPath": "transaction_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=16)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "transaction_time", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "TIME()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "tender_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_sales_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_sales_datetime", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "subTypes", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", - "type": "TRANSFORMED" - } + "typeNames": [ + "Table" ] } }, @@ -5007,7 +5000,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -5018,8 +5011,117 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "vertica", + "env": "PROD", + "database": "vmart", + "schema": "online_sales" + }, + "name": "online_sales" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:vertica" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:343f520ad0fb3259b298736800bb1385" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" } ] } @@ -5032,7 +5134,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -5043,6 +5145,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5057,12 +5160,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" } }, "systemMetadata": { @@ -5074,7 +5177,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -5084,23 +5187,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "ROS_Count": "1", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(vendor_dimension.vendor_key)", - "Projection_size": "1 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" + "create_time": "2024-06-03 12:23:45.938730+00:00", + "table_size": "6 KB" }, - "name": "vendor_dimension_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "name": "call_center_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.vendor_dimension_super", + "schemaName": "Vmart.online_sales.call_center_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -5119,7 +5216,7 @@ }, "fields": [ { - "fieldPath": "vendor_key", + "fieldPath": "call_center_key", "nullable": true, "description": "", "type": { @@ -5129,36 +5226,36 @@ }, "nativeDataType": "INTEGER()", "recursive": false, - "isPartOfKey": false + "isPartOfKey": true }, { - "fieldPath": "vendor_name", + "fieldPath": "cc_closed_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "vendor_address", + "fieldPath": "cc_open_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "vendor_city", + "fieldPath": "cc_name", "nullable": true, "description": "", "type": { @@ -5166,12 +5263,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(length=50)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "vendor_state", + "fieldPath": "cc_class", "nullable": true, "description": "", "type": { @@ -5179,70 +5276,104 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "VARCHAR(length=50)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "vendor_region", + "fieldPath": "cc_employees", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "deal_size", + "fieldPath": "cc_hours", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=20)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "last_deal_update", + "fieldPath": "cc_manager", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=40)", "recursive": false, "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Projections" + }, + { + "fieldPath": "cc_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + } + ] + } + } ] } }, @@ -5254,20 +5385,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "subTypes", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", - "type": "TRANSFORMED" - } + "typeNames": [ + "Table" ] } }, @@ -5279,7 +5403,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -5290,8 +5414,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" } ] } @@ -5304,7 +5428,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -5315,6 +5439,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5329,12 +5454,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" } }, "systemMetadata": { @@ -5346,7 +5471,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -5356,23 +5481,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "ROS_Count": "1", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(customer_dimension.customer_key)", - "Projection_size": "2119 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" + "create_time": "2024-06-03 12:23:45.935745+00:00", + "table_size": "9 KB" }, - "name": "customer_dimension_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "name": "online_page_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.customer_dimension_super", + "schemaName": "Vmart.online_sales.online_page_dimension", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -5391,7 +5510,7 @@ }, "fields": [ { - "fieldPath": "customer_key", + "fieldPath": "online_page_key", "nullable": true, "description": "", "type": { @@ -5401,62 +5520,36 @@ }, "nativeDataType": "INTEGER()", "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=16)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_name", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=256)", - "recursive": false, - "isPartOfKey": false + "isPartOfKey": true }, { - "fieldPath": "customer_gender", + "fieldPath": "start_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "title", + "fieldPath": "end_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "household_id", + "fieldPath": "page_number", "nullable": true, "description": "", "type": { @@ -5469,59 +5562,7 @@ "isPartOfKey": false }, { - "fieldPath": "customer_address", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=256)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_city", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=64)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_state", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "CHAR(length=2)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_region", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=64)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "marital_status", + "fieldPath": "page_description", "nullable": true, "description": "", "type": { @@ -5529,51 +5570,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_age", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "number_of_children", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "annual_income", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=100)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "occupation", + "fieldPath": "page_type", "nullable": true, "description": "", "type": { @@ -5581,85 +5583,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "largest_bill_amount", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_membership_card", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_since", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "deal_stage", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "deal_size", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "last_deal_update", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=100)", "recursive": false, "isPartOfKey": false } @@ -5677,38 +5601,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Projections" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", - "type": "TRANSFORMED" - } + "Table" ] } }, @@ -5720,7 +5619,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -5731,8 +5630,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" } ] } @@ -5745,7 +5644,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -5756,6 +5655,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -5770,12 +5670,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" } }, "systemMetadata": { @@ -5787,7 +5687,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -5797,23 +5697,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "ROS_Count": "1", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(employee_dimension.employee_key)", - "Projection_size": "327 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" + "create_time": "2024-06-03 12:23:45.941712+00:00", + "table_size": "182385 KB" }, - "name": "employee_dimension_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "name": "online_sales_fact", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.employee_dimension_super", + "schemaName": "Vmart.online_sales.online_sales_fact", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -5832,7 +5726,7 @@ }, "fields": [ { - "fieldPath": "employee_key", + "fieldPath": "sale_date_key", "nullable": true, "description": "", "type": { @@ -5845,72 +5739,72 @@ "isPartOfKey": false }, { - "fieldPath": "employee_gender", + "fieldPath": "ship_date_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "courtesy_title", + "fieldPath": "product_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_first_name", + "fieldPath": "product_version", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_middle_initial", + "fieldPath": "customer_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_last_name", + "fieldPath": "call_center_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_age", + "fieldPath": "online_page_key", "nullable": true, "description": "", "type": { @@ -5923,85 +5817,85 @@ "isPartOfKey": false }, { - "fieldPath": "hire_date", + "fieldPath": "shipping_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_street_address", + "fieldPath": "warehouse_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_city", + "fieldPath": "promotion_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_state", + "fieldPath": "pos_transaction_number", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "employee_region", + "fieldPath": "sales_quantity", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "job_title", + "fieldPath": "sales_dollar_amount", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "FLOAT()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "reports_to", + "fieldPath": "ship_dollar_amount", "nullable": true, "description": "", "type": { @@ -6009,12 +5903,12 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "FLOAT()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "salaried_flag", + "fieldPath": "net_dollar_amount", "nullable": true, "description": "", "type": { @@ -6022,12 +5916,12 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "FLOAT()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "annual_salary", + "fieldPath": "cost_dollar_amount", "nullable": true, "description": "", "type": { @@ -6035,12 +5929,12 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "FLOAT()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "hourly_rate", + "fieldPath": "gross_profit_dollar_amount", "nullable": true, "description": "", "type": { @@ -6053,15 +5947,41 @@ "isPartOfKey": false }, { - "fieldPath": "vacation_days", + "fieldPath": "transaction_type", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=16)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "online_sales_saledate", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "online_sales_shipdate", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false } @@ -6079,13 +5999,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Projections" + "Table" ] } }, @@ -6097,19 +6017,19 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "browsePathsV2", "aspect": { "json": { - "upstreams": [ + "path": [ { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", - "type": "TRANSFORMED" + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" } ] } @@ -6121,22 +6041,22 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "containerProperties", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", - "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" - }, - { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" - } - ] + "customProperties": { + "platform": "vertica", + "env": "PROD", + "database": "vmart", + "cluster_type": "Enterprise", + "cluster_size": "52 GB", + "subcluster": " ", + "communal_storage_path": "" + }, + "name": "vmart" } }, "systemMetadata": { @@ -6146,22 +6066,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "status", "aspect": { "json": { - "owners": [ - { - "owner": "urn:li:corpuser:dbadmin", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "removed": false } }, "systemMetadata": { @@ -6171,13 +6082,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + "platform": "urn:li:dataPlatform:vertica" } }, "systemMetadata": { @@ -6187,133 +6098,14 @@ } }, { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "ROS_Count": "1", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(warehouse_dimension.warehouse_key)", - "Projection_size": "2 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" - }, - "name": "warehouse_dimension_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.warehouse_dimension_super", - "platform": "urn:li:dataPlatform:vertica", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "warehouse_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "warehouse_name", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=20)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "warehouse_address", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=256)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "warehouse_city", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=60)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "warehouse_state", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "CHAR(length=2)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "warehouse_region", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - } - ] - } - } + "entityType": "container", + "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" ] } }, @@ -6324,15 +6116,22 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "containerProperties", "aspect": { "json": { - "typeNames": [ - "Projections" - ] + "customProperties": { + "platform": "vertica", + "env": "PROD", + "database": "vmart", + "schema": "public", + "projection_count": "12", + "udx_list": "APPROXIMATE_COUNT_DISTINCT_SYNOPSIS_INFO, APPROXIMATE_MEDIAN, APPROXIMATE_PERCENTILE, AcdDataToCount, AcdDataToLongSyn, AcdDataToSyn, AcdSynToCount, AcdSynToSyn, DelimitedExport, DelimitedExportMulti, EmptyMap, Explode, FAvroParser, FCefParser, FCsvParser, FDelimitedPairParser, FDelimitedParser, FIDXParser, FJSONParser, FRegexParser, FlexTokenizer, JsonExport, JsonExportMulti, KafkaAvroParser, KafkaCheckBrokers, KafkaExport, KafkaInsertDelimiters, KafkaInsertLengths, KafkaJsonParser, KafkaListManyTopics, KafkaListTopics, KafkaOffsets, KafkaParser, KafkaSource, KafkaTopicDetails, MSE, MapAggregate, MapAggregate, MapContainsKey, MapContainsKey, MapContainsValue, MapContainsValue, MapDelimitedExtractor, MapItems, MapItems, MapJSONExtractor, MapKeys, MapKeys, MapKeysInfo, MapKeysInfo, MapLookup, MapLookup, MapLookup, MapPut, MapRegexExtractor, MapSize, MapSize, MapToString, MapToString, MapValues, MapValues, MapValuesOrField, MapVersion, MapVersion, OrcExport, OrcExportMulti, PRC, ParquetExport, ParquetExportMulti, PickBestType, PickBestType, PickBestType, ROC, STV_AsGeoJSON, STV_AsGeoJSON, STV_AsGeoJSON, STV_Create_Index, STV_Create_Index, STV_Create_Index, STV_DWithin, STV_DWithin, STV_DWithin, STV_Describe_Index, STV_Drop_Index, STV_Export2Shapefile, STV_Extent, STV_Extent, STV_ForceLHR, STV_Geography, STV_Geography, STV_GeographyPoint, STV_Geometry, STV_Geometry, STV_GeometryPoint, STV_GeometryPoint, STV_GetExportShapefileDirectory, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_IsValidReason, STV_IsValidReason, STV_IsValidReason, STV_LineStringPoint, STV_LineStringPoint, STV_LineStringPoint, STV_MemSize, STV_MemSize, STV_MemSize, STV_NN, STV_NN, STV_NN, STV_PolygonPoint, STV_PolygonPoint, STV_PolygonPoint, STV_Refresh_Index, STV_Refresh_Index, STV_Refresh_Index, STV_Rename_Index, STV_Reverse, STV_SetExportShapefileDirectory, STV_ShpCreateTable, STV_ShpParser, STV_ShpSource, ST_Area, ST_Area, ST_Area, ST_AsBinary, ST_AsBinary, ST_AsBinary, ST_AsText, ST_AsText, ST_AsText, ST_Boundary, ST_Buffer, ST_Centroid, ST_Contains, ST_Contains, ST_Contains, ST_ConvexHull, ST_Crosses, ST_Difference, ST_Disjoint, ST_Disjoint, ST_Disjoint, ST_Distance, ST_Distance, ST_Distance, ST_Envelope, ST_Equals, ST_Equals, ST_Equals, ST_GeoHash, ST_GeoHash, ST_GeoHash, ST_GeographyFromText, ST_GeographyFromWKB, ST_GeomFromGeoHash, ST_GeomFromGeoJSON, ST_GeomFromGeoJSON, ST_GeomFromText, ST_GeomFromText, ST_GeomFromWKB, ST_GeomFromWKB, ST_GeometryN, ST_GeometryN, ST_GeometryN, ST_GeometryType, ST_GeometryType, ST_GeometryType, ST_Intersection, ST_Intersects, ST_Intersects, ST_IsEmpty, ST_IsEmpty, ST_IsEmpty, ST_IsSimple, ST_IsSimple, ST_IsSimple, ST_IsValid, ST_IsValid, ST_IsValid, ST_Length, ST_Length, ST_Length, ST_NumGeometries, ST_NumGeometries, ST_NumGeometries, ST_NumPoints, ST_NumPoints, ST_NumPoints, ST_Overlaps, ST_PointFromGeoHash, ST_PointN, ST_PointN, ST_PointN, ST_Relate, ST_SRID, ST_SRID, ST_SRID, ST_Simplify, ST_SimplifyPreserveTopology, ST_SymDifference, ST_Touches, ST_Touches, ST_Touches, ST_Transform, ST_Union, ST_Union, ST_Within, ST_Within, ST_Within, ST_X, ST_X, ST_X, ST_XMax, ST_XMax, ST_XMax, ST_XMin, ST_XMin, ST_XMin, ST_Y, ST_Y, ST_Y, ST_YMax, ST_YMax, ST_YMax, ST_YMin, ST_YMin, ST_YMin, ST_intersects, SetMapKeys, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_NumCol, Unnest, VoltageSecureAccess, VoltageSecureAccess, VoltageSecureConfigure, VoltageSecureConfigureGlobal, VoltageSecureProtect, VoltageSecureProtect, VoltageSecureProtectAllKeys, VoltageSecureRefreshPolicy, VoltageSecureVersion, append_centers, apply_bisecting_kmeans, apply_iforest, apply_inverse_pca, apply_inverse_svd, apply_kmeans, apply_kprototypes, apply_normalize, apply_one_hot_encoder, apply_pca, apply_svd, approximate_quantiles, ar_create_blobs, ar_final_newton, ar_save_model, ar_transition_newton, arima_bfgs, arima_line_search, arima_save_model, avg_all_columns_local, bisecting_kmeans_init_model, bk_apply_best_kmeans_results, bk_compute_totss_local, bk_finalize_model, bk_get_rows_in_active_cluster, bk_kmeans_compute_local_centers, bk_kmeans_compute_withinss, bk_kmeans_fast_random_init, bk_kmeans_slow_random_init, bk_kmeanspp_init_cur_cluster, bk_kmeanspp_reset_blob, bk_kmeanspp_select_new_centers, bk_kmeanspp_within_chunk_sum, bk_save_final_model, bk_write_new_cluster_level, blob_to_table, bufUdx, bufUdx, calc_pseudo_centers, calculate_alpha_linear, calculate_hessian_linear1, calculate_hessian_linear2, chi_squared, cleanup_kmeans_files, compute_and_save_global_center, compute_and_save_new_centers, compute_local_totss, compute_local_withinss, compute_new_local_centers, confusion_matrix, coordinate_descent_covariance, corr_matrix, count_rows_in_blob, create_aggregator_blob, error_rate, evaluate_naive_bayes_model, evaluate_reg_model, evaluate_svm_model, export_model_files, finalize_blob_resource_group, get_attr_minmax, get_attr_robust_zscore, get_attr_zscore, get_model_attribute, get_model_summary, get_robust_zscore_median, iforest_create_blobs, iforest_phase0_udf1, iforest_phase0_udf2, iforest_phase1_udf1, iforest_phase1_udf2, iforest_phase1_udf3, iforest_phase1_udf4, iforest_phase2_udf1, iforest_phase2_udf2, iforest_phase2_udf3, iforest_phase2_udf4, iforest_save_model, import_model_files, isOrContains, kmeansAddMetricsToModel, kmeans_init_blobs, kmeans_to_write_final_centers, lift_table, line_search_logistic1, line_search_logistic2, load_rows_into_blocks, map_factor, math_op, matrix_global_xtx, matrix_local_xtx, mode_finder, model_converter, naive_bayes_phase1, naive_bayes_phase1_blob, naive_bayes_phase2, pca_prep1_global, pca_prep1_local, pca_prep2, pmml_parser, predict_arima, predict_autoregressor, predict_linear_reg, predict_logistic_reg, predict_moving_average, predict_naive_bayes, predict_naive_bayes_classes, predict_pmml, predict_poisson_reg, predict_rf_classifier, predict_rf_classifier_classes, predict_rf_regressor, predict_svm_classifier, predict_svm_regressor, predict_xgb_classifier, predict_xgb_classifier_classes, predict_xgb_regressor, random_init, random_init_write, read_from_dfblob, read_map_factor, read_ptree, read_tree, reg_final_bfgs, reg_final_newton, reg_transition_bfgs, reg_transition_newton, reg_write_model, remove_blob, reverse_normalize, rf_blob, rf_clean, rf_phase0_udf1, rf_phase0_udf2, rf_phase1_udf1, rf_phase1_udf2, rf_phase1_udf3, rf_phase1_udf4, rf_phase2_udf1, rf_phase2_udf2, rf_phase2_udf3, rf_phase2_udf4, rf_predictor_importance, rf_save_model, rsquared, save_cv_result, save_pca_model, save_svd_model, save_svm_model, select_new_centers, store_minmax_model, store_one_hot_encoder_model, store_robust_zscore_model, store_zscore_model, table_to_blob, table_to_dfblob, tokenize, topk, update_and_return_sum_of_squared_distances, upgrade_model_format, writeInitialKmeansModelToDfs, xgb_create_blobs, xgb_phase0_udf1, xgb_phase0_udf2, xgb_phase1_udf1, xgb_phase1_udf2, xgb_phase1_udf3, xgb_phase2_udf1, xgb_phase2_udf2, xgb_phase2_udf3, xgb_predictor_importance, xgb_prune, xgb_save_model, yule_walker, ", + "udx_language": "ComplexTypesLib -- Functions for Complex Types | DelimitedExportLib -- Delimited data export package | JsonExportLib -- Json data export package | MachineLearningLib -- Machine learning package | OrcExportLib -- Orc export package | ParquetExportLib -- Parquet export package | ApproximateLib -- Approximate package | FlexTableLib -- Flexible Tables Data Load and Query | KafkaLib -- Kafka streaming load and export | PlaceLib -- Geospatial package | VoltageSecureLib -- Voltage SecureData Connector | TransformFunctions -- User-defined Python library | " + }, + "name": "public" } }, "systemMetadata": { @@ -6342,22 +6141,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "status", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", - "type": "TRANSFORMED" - } - ] + "removed": false } }, "systemMetadata": { @@ -6367,21 +6157,30 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", - "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" - }, - { - "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", - "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" - } + "platform": "urn:li:dataPlatform:vertica" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" ] } }, @@ -6391,9 +6190,25 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:343f520ad0fb3259b298736800bb1385" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -6404,6 +6219,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6418,7 +6234,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -6435,7 +6251,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -6448,20 +6264,20 @@ "ROS_Count": "1", "Projection_Type": "is_super_projection", "Is_Segmented": "True", - "Segmentation_key": "hash(shipping_dimension.shipping_key)", - "Projection_size": "1 KB", + "Segmentation_key": "hash(date_dimension.date_key)", + "Projection_size": "145 KB", "Partition_Key": "Not Available", "Number_Of_Partitions": "0", "Projection_Cached": "False" }, - "name": "shipping_dimension_super", + "name": "date_dimension_super", "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.shipping_dimension_super", + "schemaName": "Vmart.public.date_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -6480,7 +6296,7 @@ }, "fields": [ { - "fieldPath": "shipping_key", + "fieldPath": "date_key", "nullable": true, "description": "", "type": { @@ -6493,20 +6309,20 @@ "isPartOfKey": false }, { - "fieldPath": "ship_type", + "fieldPath": "date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "CHAR(length=30)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "ship_mode", + "fieldPath": "full_date_description", "nullable": true, "description": "", "type": { @@ -6514,12 +6330,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=10)", + "nativeDataType": "VARCHAR(length=18)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "ship_carrier", + "fieldPath": "day_of_week", "nullable": true, "description": "", "type": { @@ -6527,31 +6343,234 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=20)", + "nativeDataType": "VARCHAR(length=9)", "recursive": false, "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Projections" + }, + { + "fieldPath": "day_number_in_calendar_month", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_number_in_calendar_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_number_in_fiscal_month", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_number_in_fiscal_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_day_in_week_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_day_in_month_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_week_number_in_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_month_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=9)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_month_number_in_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_year_month", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=7)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_quarter", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_year_quarter", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=7)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_half_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "holiday_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=10)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weekday_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=7)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "selling_season", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + } + ] + } + } ] } }, @@ -6563,20 +6582,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension_super,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "subTypes", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", - "type": "TRANSFORMED" - } + "typeNames": [ + "Projections" ] } }, @@ -6588,7 +6600,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.date_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -6613,7 +6625,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -6624,6 +6636,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6638,7 +6651,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -6655,7 +6668,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -6668,20 +6681,20 @@ "ROS_Count": "1", "Projection_Type": "is_super_projection", "Is_Segmented": "True", - "Segmentation_key": "hash(inventory_fact.date_key, inventory_fact.product_key, inventory_fact.product_version, inventory_fact.warehouse_key, inventory_fact.qty_in_stock)", - "Projection_size": "2564 KB", + "Segmentation_key": "hash(product_dimension.product_key, product_dimension.product_version)", + "Projection_size": "19 KB", "Partition_Key": "Not Available", "Number_Of_Partitions": "0", "Projection_Cached": "False" }, - "name": "inventory_fact_super", + "name": "product_dimension_super", "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.inventory_fact_super", + "schemaName": "Vmart.public.product_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -6700,7 +6713,7 @@ }, "fields": [ { - "fieldPath": "date_key", + "fieldPath": "product_key", "nullable": true, "description": "", "type": { @@ -6713,7 +6726,7 @@ "isPartOfKey": false }, { - "fieldPath": "product_key", + "fieldPath": "product_version", "nullable": true, "description": "", "type": { @@ -6726,33 +6739,85 @@ "isPartOfKey": false }, { - "fieldPath": "product_version", + "fieldPath": "product_description", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=128)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "warehouse_key", + "fieldPath": "sku_number", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "qty_in_stock", + "fieldPath": "category_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "department_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "package_type_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "package_size", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fat_content", "nullable": true, "description": "", "type": { @@ -6765,15 +6830,158 @@ "isPartOfKey": false }, { - "fieldPath": "inventory_date", + "fieldPath": "diet_type", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight_units_of_measure", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shelf_width", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shelf_height", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shelf_depth", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_cost", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lowest_competitor_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "highest_competitor_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "average_competitor_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "discontinued_flag", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false } @@ -6791,7 +6999,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -6809,32 +7017,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.product_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -6859,7 +7042,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -6870,6 +7053,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -6884,7 +7068,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -6901,7 +7085,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -6912,22 +7096,22 @@ "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { "ROS_Count": "1", - "Projection_Type": "is_aggregate_projection, has_expressions", + "Projection_Type": "is_super_projection", "Is_Segmented": "True", - "Segmentation_key": "hash(readings.meter_id)", - "Projection_size": "0 KB", + "Segmentation_key": "hash(promotion_dimension.promotion_key)", + "Projection_size": "3 KB", "Partition_Key": "Not Available", "Number_Of_Partitions": "0", "Projection_Cached": "False" }, - "name": "readings_topk", + "name": "promotion_dimension_super", "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.readings_topk", + "schemaName": "Vmart.public.promotion_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -6946,7 +7130,7 @@ }, "fields": [ { - "fieldPath": "meter_id", + "fieldPath": "promotion_key", "nullable": true, "description": "", "type": { @@ -6959,52 +7143,151 @@ "isPartOfKey": false }, { - "fieldPath": "recent_date", + "fieldPath": "promotion_name", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "VARCHAR(length=128)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "recent_value", + "fieldPath": "price_reduction_type", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Projections" + }, + { + "fieldPath": "promotion_media_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ad_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "display_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "coupon_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ad_media_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "display_provider", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=128)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_cost", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_begin_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_end_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } ] } }, @@ -7016,20 +7299,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension_super,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "subTypes", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)", - "type": "TRANSFORMED" - } + "typeNames": [ + "Projections" ] } }, @@ -7041,7 +7317,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.readings_topk,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.promotion_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -7066,7 +7342,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -7077,6 +7353,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7091,7 +7368,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -7108,7 +7385,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -7119,22 +7396,22 @@ "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { "ROS_Count": "1", - "Projection_Type": "is_aggregate_projection, has_expressions", + "Projection_Type": "is_super_projection", "Is_Segmented": "True", - "Segmentation_key": "hash(clicks.page_id, (clicks.click_time)::date)", - "Projection_size": "0 KB", + "Segmentation_key": "hash(vendor_dimension.vendor_key)", + "Projection_size": "1 KB", "Partition_Key": "Not Available", "Number_Of_Partitions": "0", "Projection_Cached": "False" }, - "name": "clicks_agg", + "name": "vendor_dimension_super", "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.clicks_agg", + "schemaName": "Vmart.public.vendor_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -7153,7 +7430,85 @@ }, "fields": [ { - "fieldPath": "page_id", + "fieldPath": "vendor_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "deal_size", "nullable": true, "description": "", "type": { @@ -7164,6 +7519,19 @@ "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false + }, + { + "fieldPath": "last_deal_update", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false } ] } @@ -7179,7 +7547,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -7197,32 +7565,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.clicks_agg,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.vendor_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -7247,7 +7590,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -7258,6 +7601,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7272,7 +7616,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -7289,7 +7633,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -7302,20 +7646,20 @@ "ROS_Count": "1", "Projection_Type": "is_super_projection", "Is_Segmented": "True", - "Segmentation_key": "hash(phrases.phrase)", - "Projection_size": "0 KB", + "Segmentation_key": "hash(customer_dimension.customer_key)", + "Projection_size": "2119 KB", "Partition_Key": "Not Available", "Number_Of_Partitions": "0", "Projection_Cached": "False" }, - "name": "phrases_super", + "name": "customer_dimension_super", "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "public.phrases_super", + "schemaName": "Vmart.public.customer_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -7334,7 +7678,20 @@ }, "fields": [ { - "fieldPath": "phrase", + "fieldPath": "customer_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_type", "nullable": true, "description": "", "type": { @@ -7342,7 +7699,254 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(length=16)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_gender", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "title", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "household_id", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "marital_status", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_age", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number_of_children", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "annual_income", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "occupation", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "largest_bill_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_membership_card", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_since", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "deal_stage", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "deal_size", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_deal_update", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false } @@ -7360,7 +7964,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -7378,32 +7982,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.phrases_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.customer_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -7426,121 +8005,9 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "vertica", - "env": "PROD", - "database": "vmart", - "schema": "store", - "projection_count": "3", - "udx_list": "", - "udx_language": "" - }, - "name": "store" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:vertica" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Schema" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:343f520ad0fb3259b298736800bb1385" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", - "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -7551,6 +8018,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7565,12 +8033,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } }, "systemMetadata": { @@ -7582,7 +8050,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -7592,17 +8060,23 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.879951+00:00", - "table_size": "2 KB" + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "Is_Segmented": "True", + "Segmentation_key": "hash(employee_dimension.employee_key)", + "Projection_size": "327 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" }, - "name": "store_dimension", - "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "name": "employee_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "store.store_dimension", + "schemaName": "Vmart.public.employee_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -7621,7 +8095,7 @@ }, "fields": [ { - "fieldPath": "store_key", + "fieldPath": "employee_key", "nullable": true, "description": "", "type": { @@ -7631,10 +8105,10 @@ }, "nativeDataType": "INTEGER()", "recursive": false, - "isPartOfKey": true + "isPartOfKey": false }, { - "fieldPath": "store_name", + "fieldPath": "employee_gender", "nullable": true, "description": "", "type": { @@ -7642,25 +8116,25 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(length=8)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "store_number", + "fieldPath": "courtesy_title", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=8)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "store_address", + "fieldPath": "employee_first_name", "nullable": true, "description": "", "type": { @@ -7668,12 +8142,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "store_city", + "fieldPath": "employee_middle_initial", "nullable": true, "description": "", "type": { @@ -7681,12 +8155,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(length=8)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "store_state", + "fieldPath": "employee_last_name", "nullable": true, "description": "", "type": { @@ -7694,38 +8168,38 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "store_region", + "fieldPath": "employee_age", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "floor_plan_type", + "fieldPath": "hire_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "photo_processing_type", + "fieldPath": "employee_street_address", "nullable": true, "description": "", "type": { @@ -7733,12 +8207,12 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(length=256)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "financial_service_type", + "fieldPath": "employee_city", "nullable": true, "description": "", "type": { @@ -7746,64 +8220,64 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "selling_square_footage", + "fieldPath": "employee_state", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=2)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "total_square_footage", + "fieldPath": "employee_region", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "first_open_date", + "fieldPath": "job_title", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "last_remodel_date", + "fieldPath": "reports_to", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "number_of_employees", + "fieldPath": "salaried_flag", "nullable": true, "description": "", "type": { @@ -7816,7 +8290,7 @@ "isPartOfKey": false }, { - "fieldPath": "annual_shrinkage", + "fieldPath": "annual_salary", "nullable": true, "description": "", "type": { @@ -7829,7 +8303,7 @@ "isPartOfKey": false }, { - "fieldPath": "foot_traffic", + "fieldPath": "hourly_rate", "nullable": true, "description": "", "type": { @@ -7837,12 +8311,12 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "FLOAT()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "monthly_rent_cost", + "fieldPath": "vacation_days", "nullable": true, "description": "", "type": { @@ -7868,13 +8342,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "Projections" ] } }, @@ -7886,7 +8360,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.employee_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -7897,8 +8371,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -7911,7 +8385,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -7922,6 +8396,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -7936,12 +8411,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } }, "systemMetadata": { @@ -7953,7 +8428,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -7963,17 +8438,23 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.890717+00:00", - "table_size": "8646 KB" + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "Is_Segmented": "True", + "Segmentation_key": "hash(warehouse_dimension.warehouse_key)", + "Projection_size": "2 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" }, - "name": "store_orders_fact", - "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "name": "warehouse_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "store.store_orders_fact", + "schemaName": "Vmart.public.warehouse_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -7992,72 +8473,7 @@ }, "fields": [ { - "fieldPath": "product_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "product_version", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "vendor_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "employee_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "order_number", + "fieldPath": "warehouse_key", "nullable": true, "description": "", "type": { @@ -8070,72 +8486,216 @@ "isPartOfKey": false }, { - "fieldPath": "date_ordered", + "fieldPath": "warehouse_name", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=20)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "date_shipped", + "fieldPath": "warehouse_address", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=256)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "expected_delivery_date", + "fieldPath": "warehouse_city", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=60)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "date_delivered", + "fieldPath": "warehouse_state", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "CHAR(length=2)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "quantity_ordered", + "fieldPath": "warehouse_region", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false - }, + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.warehouse_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "Is_Segmented": "True", + "Segmentation_key": "hash(shipping_dimension.shipping_key)", + "Projection_size": "1 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" + }, + "name": "shipping_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "Vmart.public.shipping_dimension_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ { - "fieldPath": "quantity_delivered", + "fieldPath": "shipping_key", "nullable": true, "description": "", "type": { @@ -8148,7 +8708,7 @@ "isPartOfKey": false }, { - "fieldPath": "shipper_name", + "fieldPath": "ship_type", "nullable": true, "description": "", "type": { @@ -8156,85 +8716,33 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "unit_price", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "shipping_cost", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "total_order_cost", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "quantity_in_stock", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=30)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "reorder_level", + "fieldPath": "ship_mode", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=10)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "overstock_ceiling", + "fieldPath": "ship_carrier", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "CHAR(length=20)", "recursive": false, "isPartOfKey": false } @@ -8252,13 +8760,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "Projections" ] } }, @@ -8270,7 +8778,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.shipping_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -8281,8 +8789,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -8295,7 +8803,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -8306,6 +8814,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8320,12 +8829,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } }, "systemMetadata": { @@ -8337,7 +8846,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -8347,17 +8856,23 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.883186+00:00", - "table_size": "225060 KB" + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "Is_Segmented": "True", + "Segmentation_key": "hash(inventory_fact.date_key, inventory_fact.product_key, inventory_fact.product_version, inventory_fact.warehouse_key, inventory_fact.qty_in_stock)", + "Projection_size": "2567 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" }, - "name": "store_sales_fact", - "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "name": "inventory_fact_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "store.store_sales_fact", + "schemaName": "Vmart.public.inventory_fact_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -8415,184 +8930,41 @@ "isPartOfKey": false }, { - "fieldPath": "store_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "promotion_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "employee_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "pos_transaction_number", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sales_quantity", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sales_dollar_amount", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "cost_dollar_amount", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "gross_profit_dollar_amount", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "transaction_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=16)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "transaction_time", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "TIME()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "tender_type", + "fieldPath": "warehouse_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "store_sales_date", + "fieldPath": "qty_in_stock", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "store_sales_datetime", + "fieldPath": "inventory_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false } @@ -8610,13 +8982,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "Projections" ] } }, @@ -8628,7 +9000,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.inventory_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -8639,8 +9011,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -8653,7 +9025,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings_topk,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -8664,6 +9036,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -8678,12 +9051,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings_topk,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } }, "systemMetadata": { @@ -8695,7 +9068,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings_topk,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -8706,22 +9079,22 @@ "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { "ROS_Count": "1", - "Projection_Type": "is_super_projection", + "Projection_Type": "is_aggregate_projection, has_expressions", "Is_Segmented": "True", - "Segmentation_key": "hash(store_dimension.store_key)", - "Projection_size": "2 KB", + "Segmentation_key": "hash(readings.meter_id)", + "Projection_size": "0 KB", "Partition_Key": "Not Available", "Number_Of_Partitions": "0", "Projection_Cached": "False" }, - "name": "store_dimension_super", + "name": "readings_topk", "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "store.store_dimension_super", + "schemaName": "Vmart.public.readings_topk", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -8740,202 +9113,7 @@ }, "fields": [ { - "fieldPath": "store_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_name", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=64)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_number", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_address", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=256)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_city", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=64)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_state", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "CHAR(length=2)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_region", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=64)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "floor_plan_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "photo_processing_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "financial_service_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "selling_square_footage", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "total_square_footage", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "first_open_date", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "last_remodel_date", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "number_of_employees", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "annual_shrinkage", + "fieldPath": "meter_id", "nullable": true, "description": "", "type": { @@ -8948,20 +9126,20 @@ "isPartOfKey": false }, { - "fieldPath": "foot_traffic", + "fieldPath": "recent_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "TIMESTAMP_WITH_PRECISION()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "monthly_rent_cost", + "fieldPath": "recent_value", "nullable": true, "description": "", "type": { @@ -8969,7 +9147,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "FLOAT()", "recursive": false, "isPartOfKey": false } @@ -8987,38 +9165,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings_topk,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { - "typeNames": [ - "Projections" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", - "type": "TRANSFORMED" - } + "typeNames": [ + "Projections" ] } }, @@ -9030,7 +9183,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.readings_topk,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -9041,8 +9194,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -9055,7 +9208,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks_agg,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -9066,6 +9219,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9080,12 +9234,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks_agg,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } }, "systemMetadata": { @@ -9097,268 +9251,60 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks_agg,PROD)", "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "ROS_Count": "2", - "Projection_Type": "is_super_projection", - "Is_Segmented": "True", - "Segmentation_key": "hash(store_sales_fact.date_key, store_sales_fact.product_key, store_sales_fact.product_version, store_sales_fact.store_key, store_sales_fact.promotion_key, store_sales_fact.customer_key, store_sales_fact.employee_key, store_sales_fact.pos_transaction_number)", - "Projection_size": "225060 KB", - "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" - }, - "name": "store_sales_fact_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "store.store_sales_fact_super", - "platform": "urn:li:dataPlatform:vertica", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "date_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "product_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "product_version", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "promotion_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "employee_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "pos_transaction_number", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sales_quantity", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sales_dollar_amount", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "cost_dollar_amount", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "gross_profit_dollar_amount", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "transaction_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=16)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "transaction_time", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "TIME()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "tender_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=8)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_sales_date", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_aggregate_projection, has_expressions", + "Is_Segmented": "True", + "Segmentation_key": "hash(clicks.page_id, (clicks.click_time)::date)", + "Projection_size": "0 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" + }, + "name": "clicks_agg", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "Vmart.public.clicks_agg", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ { - "fieldPath": "store_sales_datetime", + "fieldPath": "page_id", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false } @@ -9376,7 +9322,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks_agg,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -9394,32 +9340,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.clicks_agg,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -9430,8 +9351,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -9444,7 +9365,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -9455,6 +9376,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9469,12 +9391,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } }, "systemMetadata": { @@ -9486,7 +9408,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -9499,195 +9421,39 @@ "ROS_Count": "1", "Projection_Type": "is_super_projection", "Is_Segmented": "True", - "Segmentation_key": "hash(store_orders_fact.product_key, store_orders_fact.product_version, store_orders_fact.store_key, store_orders_fact.vendor_key, store_orders_fact.employee_key, store_orders_fact.order_number, store_orders_fact.date_ordered, store_orders_fact.date_shipped)", - "Projection_size": "8646 KB", + "Segmentation_key": "hash(phrases.phrase)", + "Projection_size": "0 KB", "Partition_Key": "Not Available", - "Number_Of_Partitions": "0", - "Projection_Cached": "False" - }, - "name": "store_orders_fact_super", - "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "store.store_orders_fact_super", - "platform": "urn:li:dataPlatform:vertica", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "product_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "product_version", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "store_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "vendor_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "employee_key", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "order_number", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "date_ordered", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "date_shipped", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "expected_delivery_date", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "date_delivered", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "quantity_ordered", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "quantity_delivered", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, + "Number_Of_Partitions": "0", + "Projection_Cached": "False" + }, + "name": "phrases_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "Vmart.public.phrases_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ { - "fieldPath": "shipper_name", + "fieldPath": "phrase", "nullable": true, "description": "", "type": { @@ -9695,85 +9461,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "unit_price", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "shipping_cost", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "total_order_cost", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "quantity_in_stock", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "reorder_level", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "overstock_ceiling", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=128)", "recursive": false, "isPartOfKey": false } @@ -9791,7 +9479,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -9809,32 +9497,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.public.phrases_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -9845,8 +9508,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", - "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" } ] } @@ -9859,7 +9522,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { @@ -9868,12 +9531,12 @@ "platform": "vertica", "env": "PROD", "database": "vmart", - "schema": "online_sales", + "schema": "store", "projection_count": "3", "udx_list": "", "udx_language": "" }, - "name": "online_sales" + "name": "store" } }, "systemMetadata": { @@ -9884,7 +9547,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -9900,7 +9563,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -9916,7 +9579,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -9934,7 +9597,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -9948,30 +9611,9 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", - "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -9982,6 +9624,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -9996,12 +9639,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" } }, "systemMetadata": { @@ -10013,7 +9656,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -10023,17 +9666,23 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.900841+00:00", - "table_size": "6 KB" + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "Is_Segmented": "True", + "Segmentation_key": "hash(store_dimension.store_key)", + "Projection_size": "2 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" }, - "name": "call_center_dimension", - "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "name": "store_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "online_sales.call_center_dimension", + "schemaName": "Vmart.store.store_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -10052,46 +9701,124 @@ }, "fields": [ { - "fieldPath": "call_center_key", + "fieldPath": "store_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_region", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "VARCHAR(length=64)", "recursive": false, - "isPartOfKey": true + "isPartOfKey": false }, { - "fieldPath": "cc_closed_date", + "fieldPath": "floor_plan_type", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_open_date", + "fieldPath": "photo_processing_type", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_name", + "fieldPath": "financial_service_type", "nullable": true, "description": "", "type": { @@ -10099,25 +9826,25 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_class", + "fieldPath": "selling_square_footage", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_employees", + "fieldPath": "total_square_footage", "nullable": true, "description": "", "type": { @@ -10130,80 +9857,80 @@ "isPartOfKey": false }, { - "fieldPath": "cc_hours", + "fieldPath": "first_open_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "CHAR(length=20)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_manager", + "fieldPath": "last_remodel_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "VARCHAR(length=40)", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_address", + "fieldPath": "number_of_employees", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_city", + "fieldPath": "annual_shrinkage", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_state", + "fieldPath": "foot_traffic", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cc_region", + "fieldPath": "monthly_rent_cost", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false } @@ -10221,13 +9948,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "Projections" ] } }, @@ -10239,7 +9966,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -10250,8 +9977,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", - "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" } ] } @@ -10264,7 +9991,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -10275,6 +10002,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10289,12 +10017,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" } }, "systemMetadata": { @@ -10306,7 +10034,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -10316,17 +10044,23 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.897788+00:00", - "table_size": "9 KB" + "ROS_Count": "2", + "Projection_Type": "is_super_projection", + "Is_Segmented": "True", + "Segmentation_key": "hash(store_sales_fact.date_key, store_sales_fact.product_key, store_sales_fact.product_version, store_sales_fact.store_key, store_sales_fact.promotion_key, store_sales_fact.customer_key, store_sales_fact.employee_key, store_sales_fact.pos_transaction_number)", + "Projection_size": "225096 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" }, - "name": "online_page_dimension", - "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "name": "store_sales_fact_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "online_sales.online_page_dimension", + "schemaName": "Vmart.store.store_sales_fact_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -10345,7 +10079,7 @@ }, "fields": [ { - "fieldPath": "online_page_key", + "fieldPath": "date_key", "nullable": true, "description": "", "type": { @@ -10355,36 +10089,36 @@ }, "nativeDataType": "INTEGER()", "recursive": false, - "isPartOfKey": true + "isPartOfKey": false }, { - "fieldPath": "start_date", + "fieldPath": "product_key", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "end_date", + "fieldPath": "product_version", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "page_number", + "fieldPath": "store_key", "nullable": true, "description": "", "type": { @@ -10397,7 +10131,111 @@ "isPartOfKey": false }, { - "fieldPath": "page_description", + "fieldPath": "promotion_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pos_transaction_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sales_quantity", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sales_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "gross_profit_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "transaction_type", "nullable": true, "description": "", "type": { @@ -10405,20 +10243,59 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=100)", + "nativeDataType": "VARCHAR(length=16)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "page_type", + "fieldPath": "transaction_time", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "TIME()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "tender_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_sales_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_sales_datetime", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "VARCHAR(length=100)", + "nativeDataType": "TIMESTAMP_WITH_PRECISION()", "recursive": false, "isPartOfKey": false } @@ -10436,13 +10313,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "Projections" ] } }, @@ -10454,7 +10331,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -10465,8 +10342,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", - "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" } ] } @@ -10479,7 +10356,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -10490,6 +10367,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10504,12 +10382,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" } }, "systemMetadata": { @@ -10521,7 +10399,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -10531,17 +10409,23 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2023-10-13 11:22:37.903963+00:00", - "table_size": "182356 KB" + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "Is_Segmented": "True", + "Segmentation_key": "hash(store_orders_fact.product_key, store_orders_fact.product_version, store_orders_fact.store_key, store_orders_fact.vendor_key, store_orders_fact.employee_key, store_orders_fact.order_number, store_orders_fact.date_ordered, store_orders_fact.date_shipped)", + "Projection_size": "8646 KB", + "Partition_Key": "Not Available", + "Number_Of_Partitions": "0", + "Projection_Cached": "False" }, - "name": "online_sales_fact", - "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "name": "store_orders_fact_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", "tags": [] } }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "online_sales.online_sales_fact", + "schemaName": "Vmart.store.store_orders_fact_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -10560,7 +10444,7 @@ }, "fields": [ { - "fieldPath": "sale_date_key", + "fieldPath": "product_key", "nullable": true, "description": "", "type": { @@ -10573,7 +10457,7 @@ "isPartOfKey": false }, { - "fieldPath": "ship_date_key", + "fieldPath": "product_version", "nullable": true, "description": "", "type": { @@ -10586,7 +10470,7 @@ "isPartOfKey": false }, { - "fieldPath": "product_key", + "fieldPath": "store_key", "nullable": true, "description": "", "type": { @@ -10599,7 +10483,7 @@ "isPartOfKey": false }, { - "fieldPath": "product_version", + "fieldPath": "vendor_key", "nullable": true, "description": "", "type": { @@ -10612,7 +10496,7 @@ "isPartOfKey": false }, { - "fieldPath": "customer_key", + "fieldPath": "employee_key", "nullable": true, "description": "", "type": { @@ -10625,7 +10509,7 @@ "isPartOfKey": false }, { - "fieldPath": "call_center_key", + "fieldPath": "order_number", "nullable": true, "description": "", "type": { @@ -10638,59 +10522,59 @@ "isPartOfKey": false }, { - "fieldPath": "online_page_key", + "fieldPath": "date_ordered", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "shipping_key", + "fieldPath": "date_shipped", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "warehouse_key", + "fieldPath": "expected_delivery_date", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "promotion_key", + "fieldPath": "date_delivered", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "DATE()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "pos_transaction_number", + "fieldPath": "quantity_ordered", "nullable": true, "description": "", "type": { @@ -10703,7 +10587,7 @@ "isPartOfKey": false }, { - "fieldPath": "sales_quantity", + "fieldPath": "quantity_delivered", "nullable": true, "description": "", "type": { @@ -10716,20 +10600,20 @@ "isPartOfKey": false }, { - "fieldPath": "sales_dollar_amount", + "fieldPath": "shipper_name", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "VARCHAR(length=32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "ship_dollar_amount", + "fieldPath": "unit_price", "nullable": true, "description": "", "type": { @@ -10737,12 +10621,12 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "net_dollar_amount", + "fieldPath": "shipping_cost", "nullable": true, "description": "", "type": { @@ -10750,12 +10634,12 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cost_dollar_amount", + "fieldPath": "total_order_cost", "nullable": true, "description": "", "type": { @@ -10763,12 +10647,12 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "gross_profit_dollar_amount", + "fieldPath": "quantity_in_stock", "nullable": true, "description": "", "type": { @@ -10776,46 +10660,33 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "transaction_type", - "nullable": true, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "online_sales_saledate", + "fieldPath": "reorder_level", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "online_sales_shipdate", + "fieldPath": "overstock_ceiling", "nullable": true, "description": "", "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "INTEGER()", "recursive": false, "isPartOfKey": false } @@ -10833,13 +10704,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "Table" + "Projections" ] } }, @@ -10851,7 +10722,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.store.store_orders_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -10862,8 +10733,8 @@ "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" }, { - "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", - "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" } ] } @@ -10874,9 +10745,100 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "vertica", + "env": "PROD", + "database": "vmart", + "schema": "online_sales", + "projection_count": "3", + "udx_list": "", + "udx_language": "" + }, + "name": "online_sales" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:vertica" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:343f520ad0fb3259b298736800bb1385" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -10887,6 +10849,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -10901,7 +10864,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -10918,7 +10881,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -10944,7 +10907,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "online_sales.online_page_dimension_super", + "schemaName": "Vmart.online_sales.online_page_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -11054,7 +11017,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -11072,32 +11035,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_page_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -11122,7 +11060,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -11133,6 +11071,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -11147,7 +11086,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -11164,7 +11103,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -11190,7 +11129,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "online_sales.call_center_dimension_super", + "schemaName": "Vmart.online_sales.call_center_dimension_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -11378,7 +11317,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -11396,32 +11335,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.call_center_dimension_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -11446,7 +11360,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -11457,6 +11371,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -11471,7 +11386,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -11488,7 +11403,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact_super,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -11502,7 +11417,7 @@ "Projection_Type": "is_super_projection", "Is_Segmented": "True", "Segmentation_key": "hash(online_sales_fact.sale_date_key, online_sales_fact.ship_date_key, online_sales_fact.product_key, online_sales_fact.product_version, online_sales_fact.customer_key, online_sales_fact.call_center_key, online_sales_fact.online_page_key, online_sales_fact.shipping_key)", - "Projection_size": "182356 KB", + "Projection_size": "182385 KB", "Partition_Key": "Not Available", "Number_Of_Partitions": "0", "Projection_Cached": "False" @@ -11514,7 +11429,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "online_sales.online_sales_fact_super", + "schemaName": "Vmart.online_sales.online_sales_fact_super", "platform": "urn:li:dataPlatform:vertica", "version": 0, "created": { @@ -11806,7 +11721,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -11824,32 +11739,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "vertica-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,Vmart.online_sales.online_sales_fact_super,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { diff --git a/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml b/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml index a182e54bd53c7..6f628011279ed 100644 --- a/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml +++ b/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml @@ -11,6 +11,7 @@ source: include_models: true include_view_lineage: true include_projection_lineage: true + include_view_column_lineage: true sink: diff --git a/metadata-ingestion/tests/test_helpers/graph_helpers.py b/metadata-ingestion/tests/test_helpers/graph_helpers.py index 2e73f5e2c6cdb..127285a48d930 100644 --- a/metadata-ingestion/tests/test_helpers/graph_helpers.py +++ b/metadata-ingestion/tests/test_helpers/graph_helpers.py @@ -42,7 +42,7 @@ def import_file(self, file: Path) -> None: ) for wu in file_source.get_workunits(): if isinstance(wu, MetadataWorkUnit): - metadata = wu.get_metadata().get("metadata") + metadata = wu.metadata mcps: Iterable[ Union[ MetadataChangeProposal, @@ -111,6 +111,7 @@ def emit( UsageAggregationClass, ], callback: Union[Callable[[Exception, str], None], None] = None, + async_flag: Optional[bool] = None, ) -> None: self.emitted.append(item) # type: ignore @@ -118,7 +119,9 @@ def emit_mce(self, mce: MetadataChangeEvent) -> None: self.emitted.append(mce) def emit_mcp( - self, mcp: Union[MetadataChangeProposal, MetadataChangeProposalWrapper] + self, + mcp: Union[MetadataChangeProposal, MetadataChangeProposalWrapper], + async_flag: Optional[bool] = None, ) -> None: self.emitted.append(mcp) diff --git a/metadata-ingestion/tests/unit/api/entities/assertion/__init__.py b/metadata-ingestion/tests/unit/api/entities/assertion/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/metadata-ingestion/tests/unit/api/entities/assertion/test_assertion_config.yml b/metadata-ingestion/tests/unit/api/entities/assertion/test_assertion_config.yml new file mode 100644 index 0000000000000..a44945a30f9a3 --- /dev/null +++ b/metadata-ingestion/tests/unit/api/entities/assertion/test_assertion_config.yml @@ -0,0 +1,76 @@ +version: 1 +namespace: test-config-id-1 +assertions: + # Freshness Assertion + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.test_assertions_all_times,PROD) + type: freshness + lookback_interval: "1 hour" + last_modified_field: col_timestamp + schedule: + type: cron + cron: 0 * * * * + meta: + entity_qualified_name: TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES + entity_schema: + - col: col_date + native_type: DATE + # Volume Assertion + - type: volume + entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.test_assertions_all_times,PROD) + metric: row_count + condition: + type: less_than_or_equal_to + value: 1000 + schedule: + type: cron + cron: 0 * * * * + meta: + entity_qualified_name: TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES + entity_schema: + - col: col_date + native_type: DATE + # Field Metric Assertion + - type: field + entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.test_assertions_all_times,PROD) + field: col_date + metric: null_count + condition: + type: equal_to + value: 0 + schedule: + type: cron + cron: 0 * * * * + meta: + entity_qualified_name: TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES + entity_schema: + - col: col_date + native_type: DATE + # Field Value Assertion + - type: field + entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_event,PROD) + field: quantity + condition: + type: between + min: 0 + max: 10 + schedule: + type: on_table_change + meta: + entity_qualified_name: TEST_DB.PUBLIC.PURCHASE_EVENT + entity_schema: + - col: quantity + native_type: FLOAT + # Custom SQL Metric Assertion + - type: sql + entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_event,PROD) + statement: select mode(quantity) from test_db.public.purchase_event + condition: + type: equal_to + value: 5 + schedule: + type: on_table_change + meta: + entity_qualified_name: TEST_DB.PUBLIC.PURCHASE_EVENT + entity_schema: + - col: quantity + native_type: FLOAT diff --git a/metadata-ingestion/tests/unit/api/entities/assertion/test_assertion_config_spec.py b/metadata-ingestion/tests/unit/api/entities/assertion/test_assertion_config_spec.py new file mode 100644 index 0000000000000..74f13ac7b2a19 --- /dev/null +++ b/metadata-ingestion/tests/unit/api/entities/assertion/test_assertion_config_spec.py @@ -0,0 +1,13 @@ +from datahub.api.entities.assertion.assertion_config_spec import AssertionsConfigSpec + + +def test_assertion_config_spec_parses_correct_type(pytestconfig): + config_file = ( + pytestconfig.rootpath + / "tests/unit/api/entities/assertion/test_assertion_config.yml" + ) + + config_spec = AssertionsConfigSpec.from_yaml(config_file) + assert config_spec.version == 1 + assert config_spec.id == "test-config-id-1" + assert len(config_spec.assertions) == 5 diff --git a/metadata-ingestion/tests/unit/api/test_report.py b/metadata-ingestion/tests/unit/api/test_report.py index a885a93a0a396..acb4f055aa828 100644 --- a/metadata-ingestion/tests/unit/api/test_report.py +++ b/metadata-ingestion/tests/unit/api/test_report.py @@ -8,8 +8,8 @@ def test_report_to_string_unsampled(): source_report.report_warning("key1", "problem 1") source_report.report_failure("key2", "reason 2") str = source_report.as_string() - assert "'warnings': {'key1': ['problem 1']}" in str - assert "'failures': {'key2': ['reason 2']}" in str + assert "'warnings': [{'message': 'key1', 'context': ['problem 1']}]" in str + assert "'failures': [{'message': 'key2', 'context': ['reason 2']}]" in str def test_report_to_string_sampled(): @@ -18,7 +18,7 @@ def test_report_to_string_sampled(): source_report.report_warning(f"key{i}", "Test message") str = source_report.as_string() - assert "'sampled': '10 sampled of at most 100 entries.'" in str + assert "sampled of 100 total elements" in str str = source_report.as_json() print(str) report_as_dict = json.loads(str) diff --git a/metadata-ingestion/tests/unit/cli/assertion/__init__.py b/metadata-ingestion/tests/unit/cli/assertion/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/metadata-ingestion/tests/unit/cli/assertion/dmf_associations.sql b/metadata-ingestion/tests/unit/cli/assertion/dmf_associations.sql new file mode 100644 index 0000000000000..7e6b1982515e0 --- /dev/null +++ b/metadata-ingestion/tests/unit/cli/assertion/dmf_associations.sql @@ -0,0 +1,35 @@ + +-- Start of Assertion 025cce4dd4123c0f007908011a9c64d7 + + ALTER TABLE TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES SET DATA_METRIC_SCHEDULE = 'USING CRON 0 * * * * UTC'; + ALTER TABLE TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES ADD DATA METRIC FUNCTION test_db.datahub_dmfs.datahub__025cce4dd4123c0f007908011a9c64d7 ON (col_date); + +-- End of Assertion 025cce4dd4123c0f007908011a9c64d7 + +-- Start of Assertion 5c32eef47bd763fece7d21c7cbf6c659 + + ALTER TABLE TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES SET DATA_METRIC_SCHEDULE = 'USING CRON 0 * * * * UTC'; + ALTER TABLE TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES ADD DATA METRIC FUNCTION test_db.datahub_dmfs.datahub__5c32eef47bd763fece7d21c7cbf6c659 ON (col_date); + +-- End of Assertion 5c32eef47bd763fece7d21c7cbf6c659 + +-- Start of Assertion 04be4145bd8de10bed3dfcb0cee57842 + + ALTER TABLE TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES SET DATA_METRIC_SCHEDULE = 'USING CRON 0 * * * * UTC'; + ALTER TABLE TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES ADD DATA METRIC FUNCTION test_db.datahub_dmfs.datahub__04be4145bd8de10bed3dfcb0cee57842 ON (col_date); + +-- End of Assertion 04be4145bd8de10bed3dfcb0cee57842 + +-- Start of Assertion b065942d2bca8a4dbe90cc3ec2d9ca9f + + ALTER TABLE TEST_DB.PUBLIC.PURCHASE_EVENT SET DATA_METRIC_SCHEDULE = 'TRIGGER_ON_CHANGES'; + ALTER TABLE TEST_DB.PUBLIC.PURCHASE_EVENT ADD DATA METRIC FUNCTION test_db.datahub_dmfs.datahub__b065942d2bca8a4dbe90cc3ec2d9ca9f ON (quantity); + +-- End of Assertion b065942d2bca8a4dbe90cc3ec2d9ca9f + +-- Start of Assertion 170dbd53f28eedbbaba52ebbf189f6b1 + + ALTER TABLE TEST_DB.PUBLIC.PURCHASE_EVENT SET DATA_METRIC_SCHEDULE = 'TRIGGER_ON_CHANGES'; + ALTER TABLE TEST_DB.PUBLIC.PURCHASE_EVENT ADD DATA METRIC FUNCTION test_db.datahub_dmfs.datahub__170dbd53f28eedbbaba52ebbf189f6b1 ON (quantity); + +-- End of Assertion 170dbd53f28eedbbaba52ebbf189f6b1 diff --git a/metadata-ingestion/tests/unit/cli/assertion/dmf_definitions.sql b/metadata-ingestion/tests/unit/cli/assertion/dmf_definitions.sql new file mode 100644 index 0000000000000..85056e150b9b3 --- /dev/null +++ b/metadata-ingestion/tests/unit/cli/assertion/dmf_definitions.sql @@ -0,0 +1,71 @@ + +-- Start of Assertion 025cce4dd4123c0f007908011a9c64d7 + + CREATE or REPLACE DATA METRIC FUNCTION + test_db.datahub_dmfs.datahub__025cce4dd4123c0f007908011a9c64d7 (ARGT TABLE(col_date DATE)) + RETURNS NUMBER + COMMENT = 'Created via DataHub for assertion urn:li:assertion:025cce4dd4123c0f007908011a9c64d7 of type freshness' + AS + $$ + select case when metric <= 3600 then 1 else 0 end from (select timediff( + second, + max(col_timestamp::TIMESTAMP_LTZ), + SNOWFLAKE.CORE.DATA_METRIC_SCHEDULED_TIME() + ) as metric from TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES ) + $$; + +-- End of Assertion 025cce4dd4123c0f007908011a9c64d7 + +-- Start of Assertion 5c32eef47bd763fece7d21c7cbf6c659 + + CREATE or REPLACE DATA METRIC FUNCTION + test_db.datahub_dmfs.datahub__5c32eef47bd763fece7d21c7cbf6c659 (ARGT TABLE(col_date DATE)) + RETURNS NUMBER + COMMENT = 'Created via DataHub for assertion urn:li:assertion:5c32eef47bd763fece7d21c7cbf6c659 of type volume' + AS + $$ + select case when metric <= 1000 then 1 else 0 end from (select count(*) as metric from TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES ) + $$; + +-- End of Assertion 5c32eef47bd763fece7d21c7cbf6c659 + +-- Start of Assertion 04be4145bd8de10bed3dfcb0cee57842 + + CREATE or REPLACE DATA METRIC FUNCTION + test_db.datahub_dmfs.datahub__04be4145bd8de10bed3dfcb0cee57842 (ARGT TABLE(col_date DATE)) + RETURNS NUMBER + COMMENT = 'Created via DataHub for assertion urn:li:assertion:04be4145bd8de10bed3dfcb0cee57842 of type field' + AS + $$ + select case when metric=0 then 1 else 0 end from (select $1 as metric from (select count(*) + from TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES where col_date is null)) + $$; + +-- End of Assertion 04be4145bd8de10bed3dfcb0cee57842 + +-- Start of Assertion b065942d2bca8a4dbe90cc3ec2d9ca9f + + CREATE or REPLACE DATA METRIC FUNCTION + test_db.datahub_dmfs.datahub__b065942d2bca8a4dbe90cc3ec2d9ca9f (ARGT TABLE(quantity FLOAT)) + RETURNS NUMBER + COMMENT = 'Created via DataHub for assertion urn:li:assertion:b065942d2bca8a4dbe90cc3ec2d9ca9f of type field' + AS + $$ + select case when metric <= 0 then 1 else 0 end from (select sum($1) as metric from (select case when quantity between 0 and 10 then 0 else 1 end + from TEST_DB.PUBLIC.PURCHASE_EVENT where quantity is not null)) + $$; + +-- End of Assertion b065942d2bca8a4dbe90cc3ec2d9ca9f + +-- Start of Assertion 170dbd53f28eedbbaba52ebbf189f6b1 + + CREATE or REPLACE DATA METRIC FUNCTION + test_db.datahub_dmfs.datahub__170dbd53f28eedbbaba52ebbf189f6b1 (ARGT TABLE(quantity FLOAT)) + RETURNS NUMBER + COMMENT = 'Created via DataHub for assertion urn:li:assertion:170dbd53f28eedbbaba52ebbf189f6b1 of type sql' + AS + $$ + select case when metric=5 then 1 else 0 end from (select $1 as metric from (select mode(quantity) from test_db.public.purchase_event)) + $$; + +-- End of Assertion 170dbd53f28eedbbaba52ebbf189f6b1 diff --git a/metadata-ingestion/tests/unit/cli/assertion/test_compile.py b/metadata-ingestion/tests/unit/cli/assertion/test_compile.py new file mode 100644 index 0000000000000..47253b5b0d71e --- /dev/null +++ b/metadata-ingestion/tests/unit/cli/assertion/test_compile.py @@ -0,0 +1,42 @@ +import filecmp +import os + +from datahub.integrations.assertion.snowflake.compiler import ( + DMF_ASSOCIATIONS_FILE_NAME, + DMF_DEFINITIONS_FILE_NAME, +) +from tests.test_helpers.click_helpers import run_datahub_cmd + + +def test_compile_assertion_config_spec_for_snowflake(pytestconfig, tmp_path): + config_file = ( + pytestconfig.rootpath + / "tests/unit/api/entities/assertion/test_assertion_config.yml" + ).resolve() + + golden_file_path = pytestconfig.rootpath / "tests/unit/cli/assertion/" + run_datahub_cmd( + [ + "assertions", + "compile", + "-f", + f"{config_file}", + "-p", + "snowflake", + "-x", + "DMF_SCHEMA=test_db.datahub_dmfs", + "-o", + tmp_path, + ], + ) + + output_file_names = [ + DMF_DEFINITIONS_FILE_NAME, + DMF_ASSOCIATIONS_FILE_NAME, + ] + + for file_name in output_file_names: + assert os.path.exists(tmp_path / file_name) + assert filecmp.cmp( + golden_file_path / file_name, tmp_path / file_name + ), f"{file_name} is not as expected" diff --git a/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json index 51d5a6974eb0d..49c24d87105bb 100644 --- a/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json @@ -17,7 +17,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "statefulpipeline" } }, { @@ -32,7 +34,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "statefulpipeline" } }, { @@ -47,7 +51,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "statefulpipeline" } }, { @@ -64,7 +70,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "statefulpipeline" } }, { @@ -79,7 +87,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "statefulpipeline" } }, { @@ -220,6 +230,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -231,7 +242,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "statefulpipeline" } }, { @@ -248,7 +261,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "statefulpipeline" } }, { @@ -263,7 +278,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "statefulpipeline" } }, { @@ -283,7 +300,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "statefulpipeline" } }, { @@ -425,6 +444,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -436,7 +456,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "statefulpipeline" } }, { @@ -453,7 +475,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "statefulpipeline" } }, { @@ -468,7 +492,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "statefulpipeline" } }, { @@ -488,7 +514,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "statefulpipeline" } }, { @@ -503,7 +531,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "statefulpipeline" } }, { @@ -518,7 +548,9 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "glue-2020_04_14-07_00_00" + "runId": "glue-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided", + "pipelineName": "statefulpipeline" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json index f3d4812b79124..01ecdbb4fd331 100644 --- a/metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json @@ -10,7 +10,8 @@ "platform": "glue", "instance": "delta_platform_instance", "env": "PROD", - "database": "delta-database" + "database": "delta-database", + "CreateTime": "June 09, 2021 at 14:14:19" }, "name": "delta-database", "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database" diff --git a/metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json index 015daaa27162f..e71ede4f370ff 100644 --- a/metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json @@ -10,7 +10,8 @@ "platform": "glue", "instance": "delta_platform_instance", "env": "PROD", - "database": "delta-database" + "database": "delta-database", + "CreateTime": "June 09, 2021 at 14:14:19" }, "name": "delta-database", "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database" diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json index 7460e21e887eb..c688eb9a2d115 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json @@ -9,7 +9,11 @@ "customProperties": { "platform": "glue", "env": "PROD", - "database": "flights-database" + "database": "flights-database", + "param1": "value1", + "param2": "value2", + "LocationUri": "s3://test-bucket/test-prefix", + "CreateTime": "June 09, 2021 at 14:14:19" }, "name": "flights-database", "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database" @@ -51,6 +55,112 @@ } } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "glue", + "env": "PROD", + "database": "test-database", + "CreateTime": "June 01, 2021 at 14:55:02" + }, + "name": "test-database", + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:glue" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "glue", + "env": "PROD", + "database": "empty-database", + "CreateTime": "June 01, 2021 at 14:55:13" + }, + "name": "empty-database", + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:glue" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] + } + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -232,6 +342,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -278,58 +389,6 @@ } } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "glue", - "env": "PROD", - "database": "test-database" - }, - "name": "test-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database" - } - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:glue" - } - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Database" - ] - } - } -}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -468,6 +527,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -653,6 +713,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json index f0d506cd59f2e..de82d26c57162 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json @@ -10,7 +10,11 @@ "platform": "glue", "instance": "some_instance_name", "env": "PROD", - "database": "flights-database" + "database": "flights-database", + "param1": "value1", + "param2": "value2", + "LocationUri": "s3://test-bucket/test-prefix", + "CreateTime": "June 09, 2021 at 14:14:19" }, "name": "flights-database", "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database" @@ -53,6 +57,116 @@ } } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "glue", + "instance": "some_instance_name", + "env": "PROD", + "database": "test-database", + "CreateTime": "June 01, 2021 at 14:55:02" + }, + "name": "test-database", + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:glue", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ac4381240e82d55400c22e4392e744a4", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "glue", + "instance": "some_instance_name", + "env": "PROD", + "database": "empty-database", + "CreateTime": "June 01, 2021 at 14:55:13" + }, + "name": "empty-database", + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ac4381240e82d55400c22e4392e744a4", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ac4381240e82d55400c22e4392e744a4", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:glue", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ac4381240e82d55400c22e4392e744a4", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] + } + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -235,6 +349,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -281,60 +396,6 @@ } } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "glue", - "instance": "some_instance_name", - "env": "PROD", - "database": "test-database" - }, - "name": "test-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database" - } - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:glue", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)" - } - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Database" - ] - } - } -}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -474,6 +535,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -660,6 +722,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_add_known_query_lineage.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_add_known_query_lineage.json index bfc8a1fd16dbc..94c8947dba9ff 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_add_known_query_lineage.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_add_known_query_lineage.json @@ -18,7 +18,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:02e2ec36678bea2a8c4c855fed5255d087cfeb2710d326e95fd9b48a9c4fc0ae" + "query": "urn:li:query:6ed1d12fbf2ccc8138ceec08cc35b981030d6d004bfad9743c7afd84260fa63f" } ], "fineGrainedLineages": [ @@ -32,7 +32,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)" ], "confidenceScore": 1.0, - "query": "urn:li:query:02e2ec36678bea2a8c4c855fed5255d087cfeb2710d326e95fd9b48a9c4fc0ae" + "query": "urn:li:query:6ed1d12fbf2ccc8138ceec08cc35b981030d6d004bfad9743c7afd84260fa63f" }, { "upstreamType": "FIELD_SET", @@ -44,7 +44,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)" ], "confidenceScore": 1.0, - "query": "urn:li:query:02e2ec36678bea2a8c4c855fed5255d087cfeb2710d326e95fd9b48a9c4fc0ae" + "query": "urn:li:query:6ed1d12fbf2ccc8138ceec08cc35b981030d6d004bfad9743c7afd84260fa63f" }, { "upstreamType": "FIELD_SET", @@ -56,7 +56,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),c)" ], "confidenceScore": 1.0, - "query": "urn:li:query:02e2ec36678bea2a8c4c855fed5255d087cfeb2710d326e95fd9b48a9c4fc0ae" + "query": "urn:li:query:6ed1d12fbf2ccc8138ceec08cc35b981030d6d004bfad9743c7afd84260fa63f" } ] } @@ -64,7 +64,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:02e2ec36678bea2a8c4c855fed5255d087cfeb2710d326e95fd9b48a9c4fc0ae", + "entityUrn": "urn:li:query:6ed1d12fbf2ccc8138ceec08cc35b981030d6d004bfad9743c7afd84260fa63f", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { @@ -85,19 +85,48 @@ } } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1707182625000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "operationType": "INSERT", + "customProperties": { + "query_urn": "urn:li:query:6ed1d12fbf2ccc8138ceec08cc35b981030d6d004bfad9743c7afd84260fa63f" + }, + "lastUpdatedTimestamp": 20000 + } + } +}, { "entityType": "query", - "entityUrn": "urn:li:query:02e2ec36678bea2a8c4c855fed5255d087cfeb2710d326e95fd9b48a9c4fc0ae", + "entityUrn": "urn:li:query:6ed1d12fbf2ccc8138ceec08cc35b981030d6d004bfad9743c7afd84260fa63f", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),c)" } ] } @@ -105,7 +134,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:02e2ec36678bea2a8c4c855fed5255d087cfeb2710d326e95fd9b48a9c4fc0ae", + "entityUrn": "urn:li:query:6ed1d12fbf2ccc8138ceec08cc35b981030d6d004bfad9743c7afd84260fa63f", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -113,25 +142,5 @@ "platform": "urn:li:dataPlatform:redshift" } } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1707182625000, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "operationType": "INSERT", - "customProperties": { - "query_urn": "urn:li:query:02e2ec36678bea2a8c4c855fed5255d087cfeb2710d326e95fd9b48a9c4fc0ae" - }, - "lastUpdatedTimestamp": 20000 - } - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_basic_lineage.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_basic_lineage.json index 2eb3753473d7d..839a224a41b63 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_basic_lineage.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_basic_lineage.json @@ -81,11 +81,23 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),b)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)" } ] } diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_column_lineage_deduplication.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_column_lineage_deduplication.json index dd110a5d928df..d3ec384316818 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_column_lineage_deduplication.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_column_lineage_deduplication.json @@ -93,11 +93,23 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),b)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)" } ] } @@ -145,11 +157,29 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),c)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),b)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),c)" } ] } diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json index 5f3e81b7b9eb9..a9b5a3a7cbefa 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json @@ -106,14 +106,23 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream1,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream2,PROD)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.prod_foo,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream1,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.prod_foo,PROD),a)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream2,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.prod_foo,PROD),b)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.prod_foo,PROD),c)" } ] } diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts.json index 49458e06b0bb9..fcbe0ec5aeb83 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts.json @@ -118,11 +118,23 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream2,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream2,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream2,PROD),c)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.downstream,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream2,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.downstream,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.downstream,PROD),c)" } ] } @@ -170,11 +182,23 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream1,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream1,PROD),b)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.downstream,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream1,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.downstream,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.downstream,PROD),b)" } ] } diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json index 9567aef095f9a..48aecb9015180 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json @@ -168,9 +168,6 @@ "aspect": { "json": { "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD)" - }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.customer,PROD)" }, @@ -179,6 +176,21 @@ }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.online_survey,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),customer_id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),customer_email)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),return_date)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),return_reason)" } ] } @@ -226,14 +238,23 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.customer,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.in_person_returns,PROD)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.customer,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),customer_id)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.in_person_returns,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),customer_email)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),return_date)" } ] } diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json index fc5e5ef879fe1..9a4d405e50a7a 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json @@ -81,11 +81,23 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD),b)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),b)" } ] } @@ -184,11 +196,23 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD),b)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)" } ] } diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json index 377e3e02c970e..743e2738fc70c 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json @@ -81,11 +81,23 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),b)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)" } ] } @@ -185,11 +197,17 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_session2,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_session2,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_session2,PROD),c)" } ] } diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_view_lineage.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_view_lineage.json index bf2296c99356e..973813dae6073 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_view_lineage.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_view_lineage.json @@ -81,11 +81,23 @@ "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),b)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)" } ] } diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_utils.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_utils.py index 61b5a4dc2ffb1..744d43373a0a1 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_utils.py +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_utils.py @@ -1,5 +1,7 @@ import textwrap +from enum import Enum +import pytest import sqlglot from datahub.sql_parsing.sql_parsing_common import QueryType @@ -9,6 +11,7 @@ ) from datahub.sql_parsing.sqlglot_utils import ( generalize_query, + generalize_query_fast, get_dialect, get_query_fingerprint, is_dialect_instance, @@ -49,49 +52,67 @@ def test_query_types(): ) == (QueryType.CREATE_VIEW, {"kind": "VIEW"}) -def test_query_generalization(): - # Basic keyword normalization. - assert ( - generalize_query("select * from foo", dialect="redshift") == "SELECT * FROM foo" - ) +class QueryGeneralizationTestMode(Enum): + FULL = "full" + FAST = "fast" + BOTH = "both" - # Comment removal and whitespace normalization. - assert ( - generalize_query( - "/* query system = foo, id = asdf */\nselect /* inline comment */ *\nfrom foo", - dialect="redshift", - ) - == "SELECT * FROM foo" - ) - # Parameter normalization. - assert ( - generalize_query( +@pytest.mark.parametrize( + "query, dialect, expected, mode", + [ + # Basic keyword normalization. + ( + "select * from foo", + "redshift", + "SELECT * FROM foo", + QueryGeneralizationTestMode.FULL, + ), + # Comment removal and whitespace normalization. + ( + "/* query system = foo, id = asdf */\nSELECT /* inline comment */ *\nFROM foo", + "redshift", + "SELECT * FROM foo", + QueryGeneralizationTestMode.BOTH, + ), + # Parameter normalization. + ( "UPDATE \"books\" SET page_count = page_count + 1, author_count = author_count + 1 WHERE book_title = 'My New Book'", - dialect="redshift", - ) - == 'UPDATE "books" SET page_count = page_count + ?, author_count = author_count + ? WHERE book_title = ?' - ) - assert ( - generalize_query( - "select * from foo where date = '2021-01-01'", dialect="redshift" - ) - == "SELECT * FROM foo WHERE date = ?" - ) - assert ( - generalize_query( - "select * from books where category in ('fiction', 'biography', 'fantasy')", - dialect="redshift", - ) - == "SELECT * FROM books WHERE category IN (?)" - ) - assert ( - generalize_query( + "redshift", + 'UPDATE "books" SET page_count = page_count + ?, author_count = author_count + ? WHERE book_title = ?', + QueryGeneralizationTestMode.BOTH, + ), + ( + "SELECT * FROM foo WHERE date = '2021-01-01'", + "redshift", + "SELECT * FROM foo WHERE date = ?", + QueryGeneralizationTestMode.BOTH, + ), + ( + "SELECT * FROM books WHERE category IN ('fiction', 'biography', 'fantasy')", + "redshift", + "SELECT * FROM books WHERE category IN (?)", + QueryGeneralizationTestMode.BOTH, + ), + ( + textwrap.dedent( + """\ + INSERT INTO MyTable + (Column1, Column2, Column3) + VALUES + ('John', 123, 'Lloyds Office'); + """ + ), + "mssql", + "INSERT INTO MyTable (Column1, Column2, Column3) VALUES (?)", + QueryGeneralizationTestMode.BOTH, + ), + ( textwrap.dedent( """\ /* Copied from https://stackoverflow.com/a/452934/5004662 */ INSERT INTO MyTable - ( Column1, Column2, Column3 ) + (Column1, Column2, Column3) VALUES /* multiple value rows */ ('John', 123, 'Lloyds Office'), @@ -100,10 +121,42 @@ def test_query_generalization(): ('Miranda', 126, 'Bristol Office'); """ ), - dialect="mssql", + "mssql", + "INSERT INTO MyTable (Column1, Column2, Column3) VALUES (?), (?), (?), (?)", + QueryGeneralizationTestMode.FULL, + ), + # Test table name normalization. + # These are only supported with fast normalization. + ( + "SELECT * FROM datahub_community.fivetran_interval_unconstitutional_staging.datahub_slack_mess-staging-480fd5a7-58f4-4cc9-b6fb-87358788efe6", + "bigquery", + "SELECT * FROM datahub_community.fivetran_interval_unconstitutional_staging.datahub_slack_mess-staging-00000000-0000-0000-0000-000000000000", + QueryGeneralizationTestMode.FAST, + ), + ( + "SELECT * FROM datahub_community.maggie.commonroom_slack_members_20240315", + "bigquery", + "SELECT * FROM datahub_community.maggie.commonroom_slack_members_YYYYMMDD", + QueryGeneralizationTestMode.FAST, + ), + ( + "SELECT COUNT(*) FROM ge_temp_aa91f1fd", + "bigquery", + "SELECT COUNT(*) FROM ge_temp_abcdefgh", + QueryGeneralizationTestMode.FAST, + ), + ], +) +def test_query_generalization( + query: str, dialect: str, expected: str, mode: QueryGeneralizationTestMode +) -> None: + if mode in {QueryGeneralizationTestMode.FULL, QueryGeneralizationTestMode.BOTH}: + assert generalize_query(query, dialect=dialect) == expected + if mode in {QueryGeneralizationTestMode.FAST, QueryGeneralizationTestMode.BOTH}: + assert ( + generalize_query_fast(query, dialect=dialect, change_table_names=True) + == expected ) - == "INSERT INTO MyTable (Column1, Column2, Column3) VALUES (?), (?), (?), (?)" - ) def test_query_fingerprint(): diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state.json b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state.json index fcf73d9614f24..ce03804279097 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state.json +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state.json @@ -17,7 +17,7 @@ "state": { "formatVersion": "1.0", "serde": "utf-8", - "payload": "{\"urns\": [\"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)\"]}" + "payload": "{\"urns\": [\"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)\", \"urn:li:dataProcessInstance:478810e859f870a54f72c681f41af619\"]}" }, "runId": "dummy-test-stateful-ingestion" } diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_after_deleted.json b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_after_deleted.json index 5477af72a1939..6a00e67a2ca21 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_after_deleted.json +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_checkpoint_state_after_deleted.json @@ -17,7 +17,7 @@ "state": { "formatVersion": "1.0", "serde": "utf-8", - "payload": "{\"urns\": [\"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)\"]}" + "payload": "{\"urns\": [\"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)\", \"urn:li:dataProcessInstance:7f26c3b4d2d82ace47f4b9dd0c9dea26\"]}" }, "runId": "dummy-test-stateful-ingestion" } diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion.json b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion.json index 4a77651c93066..adf11a2833914 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion.json +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion.json @@ -46,5 +46,43 @@ "runId": "dummy-test-stateful-ingestion", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:478810e859f870a54f72c681f41af619", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "job1", + "type": "BATCH_SCHEDULED", + "created": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "dummy-test-stateful-ingestion", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:478810e859f870a54f72c681f41af619", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "dummy-test-stateful-ingestion", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion_after_deleted.json b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion_after_deleted.json index 9d6f755374462..e4893642d61ae 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion_after_deleted.json +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/golden_test_stateful_ingestion_after_deleted.json @@ -31,6 +31,44 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:7f26c3b4d2d82ace47f4b9dd0c9dea26", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "job2", + "type": "BATCH_SCHEDULED", + "created": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "dummy-test-stateful-ingestion", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:7f26c3b4d2d82ace47f4b9dd0c9dea26", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "dummy-test-stateful-ingestion", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)", diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_redundant_run_skip_handler.py b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_redundant_run_skip_handler.py index 0400bd6a72aa5..be6efd3e121ff 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_redundant_run_skip_handler.py +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_redundant_run_skip_handler.py @@ -1,4 +1,5 @@ from datetime import datetime, timezone +from typing import Iterable from unittest import mock import pytest @@ -24,7 +25,7 @@ @pytest.fixture -def stateful_source(mock_datahub_graph: DataHubGraph) -> SnowflakeV2Source: +def stateful_source(mock_datahub_graph: DataHubGraph) -> Iterable[SnowflakeV2Source]: pipeline_name = "test_redundant_run_lineage" run_id = "test_redundant_run" ctx = PipelineContext( @@ -43,8 +44,9 @@ def stateful_source(mock_datahub_graph: DataHubGraph) -> SnowflakeV2Source: ), ), ) - source = SnowflakeV2Source(ctx=ctx, config=config) - return source + + with mock.patch("snowflake.connector.connect"): + yield SnowflakeV2Source(ctx=ctx, config=config) def test_redundant_run_job_ids(stateful_source: SnowflakeV2Source) -> None: diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py index 50d9b86b3a017..e3a2a6cccea79 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py @@ -1,3 +1,4 @@ +import time from dataclasses import dataclass, field as dataclass_field from typing import Any, Dict, Iterable, List, Optional, cast from unittest import mock @@ -7,6 +8,7 @@ from freezegun import freeze_time from pydantic import Field +from datahub.api.entities.dataprocess.dataprocess_instance import DataProcessInstance from datahub.configuration.common import AllowDenyPattern from datahub.configuration.source_common import DEFAULT_ENV, DatasetSourceConfigMixin from datahub.emitter.mcp import MetadataChangeProposalWrapper @@ -24,7 +26,10 @@ StatefulIngestionConfigBase, StatefulIngestionSourceBase, ) -from datahub.metadata.schema_classes import StatusClass +from datahub.metadata.com.linkedin.pegasus2avro.dataprocess import ( + DataProcessInstanceProperties, +) +from datahub.metadata.schema_classes import AuditStampClass, StatusClass from datahub.utilities.urns.dataset_urn import DatasetUrn from tests.test_helpers import mce_helpers from tests.test_helpers.state_helpers import ( @@ -62,6 +67,10 @@ class DummySourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin): default=False, description="Should this dummy source report a failure.", ) + dpi_id_to_ingest: Optional[str] = Field( + default=None, + description="Data process instance id to ingest.", + ) class DummySource(StatefulIngestionSourceBase): @@ -109,6 +118,24 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: aspect=StatusClass(removed=False), ).as_workunit() + if self.source_config.dpi_id_to_ingest: + dpi = DataProcessInstance( + id=self.source_config.dpi_id_to_ingest, + orchestrator="dummy", + ) + + yield MetadataChangeProposalWrapper( + entityUrn=str(dpi.urn), + aspect=DataProcessInstanceProperties( + name=dpi.id, + created=AuditStampClass( + time=int(time.time() * 1000), + actor="urn:li:corpuser:datahub", + ), + type=dpi.type, + ), + ).as_workunit() + if self.source_config.report_failure: self.reporter.report_failure("Dummy error", "Error") @@ -152,6 +179,7 @@ def test_stateful_ingestion(pytestconfig, tmp_path, mock_time): "stateful_ingestion": { "enabled": True, "remove_stale_metadata": True, + "fail_safe_threshold": 100, "state_provider": { "type": "file", "config": { @@ -159,6 +187,7 @@ def test_stateful_ingestion(pytestconfig, tmp_path, mock_time): }, }, }, + "dpi_id_to_ingest": "job1", }, }, "sink": { @@ -207,6 +236,7 @@ def test_stateful_ingestion(pytestconfig, tmp_path, mock_time): pipeline_run2_config["source"]["config"]["dataset_patterns"] = { "allow": ["dummy_dataset1", "dummy_dataset2"], } + pipeline_run2_config["source"]["config"]["dpi_id_to_ingest"] = "job2" pipeline_run2_config["sink"]["config"][ "filename" ] = f"{tmp_path}/{output_file_name_after_deleted}" @@ -253,6 +283,16 @@ def test_stateful_ingestion(pytestconfig, tmp_path, mock_time): ] assert sorted(deleted_dataset_urns) == sorted(difference_dataset_urns) + report = pipeline_run2.source.get_report() + assert isinstance(report, StaleEntityRemovalSourceReport) + # assert report last ingestion state non_deletable entity urns + non_deletable_urns: List[str] = [ + "urn:li:dataProcessInstance:478810e859f870a54f72c681f41af619", + ] + assert sorted(non_deletable_urns) == sorted( + report.last_state_non_deletable_entities + ) + @freeze_time(FROZEN_TIME) def test_stateful_ingestion_failure(pytestconfig, tmp_path, mock_time): diff --git a/metadata-ingestion/tests/unit/test_bigquery_source.py b/metadata-ingestion/tests/unit/test_bigquery_source.py index c501593fbed01..b58f35c0deef5 100644 --- a/metadata-ingestion/tests/unit/test_bigquery_source.py +++ b/metadata-ingestion/tests/unit/test_bigquery_source.py @@ -235,10 +235,10 @@ def test_get_dataplatform_instance_aspect_returns_project_id(get_bq_client_mock) data_platform_instance = source.get_dataplatform_instance_aspect( "urn:li:test", project_id ) - metadata = data_platform_instance.get_metadata()["metadata"] + metadata = data_platform_instance.metadata - assert data_platform_instance is not None - assert metadata.aspectName == "dataPlatformInstance" + assert isinstance(metadata, MetadataChangeProposalWrapper) + assert isinstance(metadata.aspect, DataPlatformInstanceClass) assert metadata.aspect.instance == expected_instance @@ -250,10 +250,10 @@ def test_get_dataplatform_instance_default_no_instance(get_bq_client_mock): data_platform_instance = source.get_dataplatform_instance_aspect( "urn:li:test", "project_id" ) - metadata = data_platform_instance.get_metadata()["metadata"] + metadata = data_platform_instance.metadata - assert data_platform_instance is not None - assert metadata.aspectName == "dataPlatformInstance" + assert isinstance(metadata, MetadataChangeProposalWrapper) + assert isinstance(metadata.aspect, DataPlatformInstanceClass) assert metadata.aspect.instance is None @@ -820,6 +820,7 @@ def bigquery_view_1() -> BigqueryView: comment="comment1", view_definition="CREATE VIEW 1", materialized=False, + labels=None, ) @@ -833,6 +834,7 @@ def bigquery_view_2() -> BigqueryView: comment="comment2", view_definition="CREATE VIEW 2", materialized=True, + labels=None, ) diff --git a/metadata-ingestion/tests/unit/test_dbt_source.py b/metadata-ingestion/tests/unit/test_dbt_source.py index 0206f2e280ef2..99387ab4e6ae4 100644 --- a/metadata-ingestion/tests/unit/test_dbt_source.py +++ b/metadata-ingestion/tests/unit/test_dbt_source.py @@ -148,7 +148,7 @@ def test_dbt_source_patching_tags(): ["new_non_dbt", "dbt:new_dbt"] ) transformed_tags = source.get_transformed_tags_by_prefix( - new_tag_aspect.tags, "urn:li:dataset:dummy", "urn:li:tag:dbt:" + new_tag_aspect.tags, "urn:li:dataset:dummy", "dbt:" ) expected_tags = { "urn:li:tag:new_non_dbt", diff --git a/metadata-ingestion/tests/unit/test_glue_source.py b/metadata-ingestion/tests/unit/test_glue_source.py index 5e721fc5c1293..c8b7e021cf5a0 100644 --- a/metadata-ingestion/tests/unit/test_glue_source.py +++ b/metadata-ingestion/tests/unit/test_glue_source.py @@ -135,6 +135,11 @@ def test_glue_ingest( get_tables_response_2, {"DatabaseName": "test-database"}, ) + glue_stubber.add_response( + "get_tables", + {"TableList": []}, + {"DatabaseName": "empty-database"}, + ) glue_stubber.add_response("get_jobs", get_jobs_response, {}) glue_stubber.add_response( "get_dataflow_graph", diff --git a/metadata-ingestion/tests/unit/test_glue_source_stubs.py b/metadata-ingestion/tests/unit/test_glue_source_stubs.py index c971001f97072..fc4c9e91410e0 100644 --- a/metadata-ingestion/tests/unit/test_glue_source_stubs.py +++ b/metadata-ingestion/tests/unit/test_glue_source_stubs.py @@ -61,6 +61,8 @@ } ], "CatalogId": "123412341234", + "LocationUri": "s3://test-bucket/test-prefix", + "Parameters": {"param1": "value1", "param2": "value2"}, }, { "Name": "test-database", @@ -75,6 +77,19 @@ ], "CatalogId": "123412341234", }, + { + "Name": "empty-database", + "CreateTime": datetime.datetime(2021, 6, 1, 14, 55, 13), + "CreateTableDefaultPermissions": [ + { + "Principal": { + "DataLakePrincipalIdentifier": "IAM_ALLOWED_PRINCIPALS" + }, + "Permissions": ["ALL"], + } + ], + "CatalogId": "123412341234", + }, ] } databases_1 = { diff --git a/metadata-ingestion/tests/unit/test_iceberg.py b/metadata-ingestion/tests/unit/test_iceberg.py index e2b463004f5a1..c8c6c6ac8a85d 100644 --- a/metadata-ingestion/tests/unit/test_iceberg.py +++ b/metadata-ingestion/tests/unit/test_iceberg.py @@ -1,6 +1,6 @@ import uuid from decimal import Decimal -from typing import Any, Optional +from typing import Any, Dict, List, Optional import pytest from pydantic import ValidationError @@ -34,7 +34,6 @@ IcebergSource, IcebergSourceConfig, ) -from datahub.ingestion.source.iceberg.iceberg_common import IcebergCatalogConfig from datahub.metadata.com.linkedin.pegasus2avro.schema import ArrayType, SchemaField from datahub.metadata.schema_classes import ( ArrayTypeClass, @@ -50,9 +49,7 @@ def with_iceberg_source() -> IcebergSource: - catalog: IcebergCatalogConfig = IcebergCatalogConfig( - name="test", type="rest", config={} - ) + catalog = {"test": {"type": "rest"}} return IcebergSource( ctx=PipelineContext(run_id="iceberg-source-test"), config=IcebergSourceConfig(catalog=catalog), @@ -95,14 +92,29 @@ def test_config_catalog_not_configured(): """ Test when an Iceberg catalog is provided, but not properly configured. """ + # When no catalog configurationis provided, the config should be invalid + with pytest.raises(ValidationError, match="type"): + IcebergSourceConfig(catalog={}) # type: ignore + + # When a catalog name is provided without configuration, the config should be invalid with pytest.raises(ValidationError): - IcebergCatalogConfig() # type: ignore + IcebergSourceConfig(catalog={"test": {}}) - with pytest.raises(ValidationError, match="conf"): - IcebergCatalogConfig(type="a type") # type: ignore - with pytest.raises(ValidationError, match="type"): - IcebergCatalogConfig(conf={}) # type: ignore +def test_config_deprecated_catalog_configuration(): + """ + Test when a deprecated Iceberg catalog configuration is provided, it should be converted to the current scheme. + """ + deprecated_config = { + "name": "test", + "type": "rest", + "config": {"uri": "http://a.uri.test", "another_prop": "another_value"}, + } + migrated_config = IcebergSourceConfig(catalog=deprecated_config) + assert migrated_config.catalog["test"] is not None + assert migrated_config.catalog["test"]["type"] == "rest" + assert migrated_config.catalog["test"]["uri"] == "http://a.uri.test" + assert migrated_config.catalog["test"]["another_prop"] == "another_value" def test_config_for_tests(): @@ -112,6 +124,34 @@ def test_config_for_tests(): with_iceberg_source() +def test_config_support_nested_dicts(): + """ + Test that Iceberg source supports nested dictionaries inside its configuration, as allowed by pyiceberg. + """ + catalog = { + "test": { + "type": "rest", + "nested_dict": { + "nested_key": "nested_value", + "nested_array": ["a1", "a2"], + "subnested_dict": {"subnested_key": "subnested_value"}, + }, + } + } + test_config = IcebergSourceConfig(catalog=catalog) + assert isinstance(test_config.catalog["test"]["nested_dict"], Dict) + assert test_config.catalog["test"]["nested_dict"]["nested_key"] == "nested_value" + assert isinstance(test_config.catalog["test"]["nested_dict"]["nested_array"], List) + assert test_config.catalog["test"]["nested_dict"]["nested_array"][0] == "a1" + assert isinstance( + test_config.catalog["test"]["nested_dict"]["subnested_dict"], Dict + ) + assert ( + test_config.catalog["test"]["nested_dict"]["subnested_dict"]["subnested_key"] + == "subnested_value" + ) + + @pytest.mark.parametrize( "iceberg_type, expected_schema_field_type", [ diff --git a/metadata-ingestion/tests/unit/test_kafka_source.py b/metadata-ingestion/tests/unit/test_kafka_source.py index 5ad9ac45534aa..b4e37d288a304 100644 --- a/metadata-ingestion/tests/unit/test_kafka_source.py +++ b/metadata-ingestion/tests/unit/test_kafka_source.py @@ -334,7 +334,9 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: mock_kafka_consumer.assert_called_once() mock_kafka_instance.list_topics.assert_called_once() - assert len(workunits) == 8 + # Along with with 4 topics(3 with schema and 1 schemaless) which constitutes to 8 workunits, + # there will be 6 schemas (1 key and 1 value schema for 3 topics) which constitutes to 12 workunits + assert len(workunits) == 20 i: int = -1 for wu in workunits: assert isinstance(wu, MetadataWorkUnit) @@ -343,6 +345,8 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: mce: MetadataChangeEvent = wu.metadata i += 1 + # Only topic (named schema_less_topic) does not have schema metadata but other workunits (that are created + # for schema) will have corresponding SchemaMetadata aspect if i < len(topic_subject_schema_map.keys()): # First 3 workunits (topics) must have schemaMetadata aspect assert isinstance(mce.proposedSnapshot.aspects[1], SchemaMetadataClass) @@ -380,11 +384,18 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: ) # Make sure we have 2 fields, one from the key schema & one from the value schema. assert len(schemaMetadataAspect.fields) == 2 - else: + elif i == len(topic_subject_schema_map.keys()): # Last topic('schema_less_topic') has no schema defined in the registry. # The schemaMetadata aspect should not be present for this. for aspect in mce.proposedSnapshot.aspects: assert not isinstance(aspect, SchemaMetadataClass) + else: + # Last 2 workunits (schemas) must have schemaMetadata aspect + assert isinstance(mce.proposedSnapshot.aspects[1], SchemaMetadataClass) + schemaMetadataAspectObj: SchemaMetadataClass = mce.proposedSnapshot.aspects[ + 1 + ] + assert isinstance(schemaMetadataAspectObj.platformSchema, KafkaSchemaClass) @pytest.mark.parametrize( @@ -465,7 +476,7 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: workunits = list(kafka_source.get_workunits()) - assert len(workunits) == 2 + assert len(workunits) == 6 if ignore_warnings_on_schema_type: assert not kafka_source.report.warnings else: @@ -643,8 +654,10 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: }, ctx, ) + # Along with with 1 topics(and 5 meta mapping) it constitutes to 6 workunits, + # there will be 2 schemas which constitutes to 4 workunits (1 mce and 1 mcp each) workunits = [w for w in kafka_source.get_workunits()] - assert len(workunits) == 6 + assert len(workunits) == 10 mce = workunits[0].metadata assert isinstance(mce, MetadataChangeEvent) @@ -677,11 +690,49 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: "urn:li:glossaryTerm:double_meta_property", ] ) - assert isinstance(workunits[2].metadata, MetadataChangeProposalWrapper) + assert isinstance(workunits[1].metadata, MetadataChangeProposalWrapper) + mce = workunits[2].metadata + assert isinstance(mce, MetadataChangeEvent) assert isinstance(workunits[3].metadata, MetadataChangeProposalWrapper) - assert isinstance(workunits[4].metadata, MetadataChangeProposalWrapper) + + mce = workunits[4].metadata + assert isinstance(mce, MetadataChangeEvent) + ownership_aspect = [ + asp for asp in mce.proposedSnapshot.aspects if isinstance(asp, OwnershipClass) + ][0] + assert ownership_aspect == make_ownership_aspect_from_urn_list( + [ + make_owner_urn("charles", OwnerType.USER), + make_owner_urn("jdoe.last@gmail.com", OwnerType.USER), + ], + "SERVICE", + ) + + tags_aspect = [ + asp for asp in mce.proposedSnapshot.aspects if isinstance(asp, GlobalTagsClass) + ][0] + assert tags_aspect == make_global_tag_aspect_with_tag_list( + ["has_pii_test", "int_meta_property"] + ) + + terms_aspect = [ + asp + for asp in mce.proposedSnapshot.aspects + if isinstance(asp, GlossaryTermsClass) + ][0] + assert terms_aspect == make_glossary_terms_aspect_from_urn_list( + [ + "urn:li:glossaryTerm:Finance_test", + "urn:li:glossaryTerm:double_meta_property", + ] + ) + assert isinstance(workunits[5].metadata, MetadataChangeProposalWrapper) - assert workunits[2].metadata.aspectName == "glossaryTermKey" - assert workunits[3].metadata.aspectName == "glossaryTermKey" - assert workunits[4].metadata.aspectName == "tagKey" - assert workunits[5].metadata.aspectName == "tagKey" + assert isinstance(workunits[6].metadata, MetadataChangeProposalWrapper) + assert isinstance(workunits[7].metadata, MetadataChangeProposalWrapper) + assert isinstance(workunits[8].metadata, MetadataChangeProposalWrapper) + assert isinstance(workunits[9].metadata, MetadataChangeProposalWrapper) + assert workunits[6].metadata.aspectName == "glossaryTermKey" + assert workunits[7].metadata.aspectName == "glossaryTermKey" + assert workunits[8].metadata.aspectName == "tagKey" + assert workunits[9].metadata.aspectName == "tagKey" diff --git a/metadata-ingestion/tests/unit/test_mlflow_source.py b/metadata-ingestion/tests/unit/test_mlflow_source.py index 374816055b216..ae5a42bad229d 100644 --- a/metadata-ingestion/tests/unit/test_mlflow_source.py +++ b/metadata-ingestion/tests/unit/test_mlflow_source.py @@ -72,7 +72,7 @@ def test_stages(source): None, } workunits = source._get_tags_workunits() - names = [wu.get_metadata()["metadata"].aspect.name for wu in workunits] + names = [wu.metadata.aspect.name for wu in workunits] assert len(names) == len(mlflow_registered_model_stages) assert set(names) == { @@ -100,7 +100,7 @@ def test_model_without_run(source, registered_model, model_version): model_version=model_version, run=run, ) - aspect = wu.get_metadata()["metadata"].aspect + aspect = wu.metadata.aspect assert aspect.hyperParams is None assert aspect.trainingMetrics is None diff --git a/metadata-ingestion/tests/unit/test_nifi_source.py b/metadata-ingestion/tests/unit/test_nifi_source.py index 26c803927f9e6..9e8bf64261ffa 100644 --- a/metadata-ingestion/tests/unit/test_nifi_source.py +++ b/metadata-ingestion/tests/unit/test_nifi_source.py @@ -333,9 +333,10 @@ def test_single_user_auth_failed_to_get_token(): list(source.get_workunits()) assert source.get_report().failures - assert "Failed to authenticate" in list( - source.get_report().failures[config.site_url] - ) + + assert "Failed to authenticate" in [ + failure.message for failure in source.get_report().failures + ] def test_kerberos_auth_failed_to_get_token(): @@ -352,9 +353,9 @@ def test_kerberos_auth_failed_to_get_token(): list(source.get_workunits()) assert source.get_report().failures - assert "Failed to authenticate" in list( - source.get_report().failures[config.site_url] - ) + assert "Failed to authenticate" in [ + failure.message for failure in source.get_report().failures + ] def test_client_cert_auth_failed(): @@ -372,9 +373,9 @@ def test_client_cert_auth_failed(): list(source.get_workunits()) assert source.get_report().failures - assert "Failed to authenticate" in list( - source.get_report().failures[config.site_url] - ) + assert "Failed to authenticate" in [ + failure.message for failure in source.get_report().failures + ] def test_failure_to_create_nifi_flow(): @@ -392,9 +393,9 @@ def test_failure_to_create_nifi_flow(): list(source.get_workunits()) assert source.get_report().failures - assert "Failed to get root process group flow" in list( - source.get_report().failures[config.site_url] - ) + assert "Failed to get root process group flow" in [ + failure.message for failure in source.get_report().failures + ] def test_site_url_no_context(): diff --git a/metadata-ingestion/tests/unit/test_pipeline.py b/metadata-ingestion/tests/unit/test_pipeline.py index 851350933f59e..01403163bbfbd 100644 --- a/metadata-ingestion/tests/unit/test_pipeline.py +++ b/metadata-ingestion/tests/unit/test_pipeline.py @@ -31,15 +31,16 @@ class TestPipeline: + @patch("confluent_kafka.Consumer", autospec=True) @patch("datahub.ingestion.source.kafka.KafkaSource.get_workunits", autospec=True) @patch("datahub.ingestion.sink.console.ConsoleSink.close", autospec=True) @freeze_time(FROZEN_TIME) - def test_configure(self, mock_sink, mock_source): + def test_configure(self, mock_sink, mock_source, mock_consumer): pipeline = Pipeline.create( { "source": { "type": "kafka", - "config": {"connection": {"bootstrap": "localhost:9092"}}, + "config": {"connection": {"bootstrap": "fake-dns-name:9092"}}, }, "sink": {"type": "console"}, } diff --git a/metadata-ingestion/tests/unit/test_plugin_system.py b/metadata-ingestion/tests/unit/test_plugin_system.py index 4d1ebce2be849..0e12416325bf9 100644 --- a/metadata-ingestion/tests/unit/test_plugin_system.py +++ b/metadata-ingestion/tests/unit/test_plugin_system.py @@ -7,6 +7,7 @@ from datahub.ingestion.api.registry import PluginRegistry from datahub.ingestion.api.sink import Sink from datahub.ingestion.extractor.extractor_registry import extractor_registry +from datahub.ingestion.fs.fs_registry import fs_registry from datahub.ingestion.reporting.reporting_provider_registry import ( reporting_provider_registry, ) @@ -54,6 +55,7 @@ (reporting_provider_registry, ["datahub", "file"]), (ingestion_checkpoint_provider_registry, ["datahub"]), (lite_registry, ["duckdb"]), + (fs_registry, ["file", "http", "s3"]), ], ) def test_registry_defaults(registry: PluginRegistry, expected: List[str]) -> None: diff --git a/metadata-ingestion/tests/unit/test_redash_source.py b/metadata-ingestion/tests/unit/test_redash_source.py index 925c41689c6cc..2982fe76c4d4e 100644 --- a/metadata-ingestion/tests/unit/test_redash_source.py +++ b/metadata-ingestion/tests/unit/test_redash_source.py @@ -489,9 +489,7 @@ def test_get_dashboard_snapshot_before_v10(): ], datasets=[], lastModified=ChangeAuditStamps( - created=AuditStamp( - time=1628882055288, actor="urn:li:corpuser:unknown" - ), + created=None, lastModified=AuditStamp( time=1628882055288, actor="urn:li:corpuser:unknown" ), @@ -521,9 +519,7 @@ def test_get_dashboard_snapshot_after_v10(): ], datasets=[], lastModified=ChangeAuditStamps( - created=AuditStamp( - time=1628882055288, actor="urn:li:corpuser:unknown" - ), + created=None, lastModified=AuditStamp( time=1628882055288, actor="urn:li:corpuser:unknown" ), @@ -551,9 +547,7 @@ def test_get_known_viz_chart_snapshot(mocked_data_source): title="My Query Chart", description="", lastModified=ChangeAuditStamps( - created=AuditStamp( - time=1628882022544, actor="urn:li:corpuser:unknown" - ), + created=None, lastModified=AuditStamp( time=1628882022544, actor="urn:li:corpuser:unknown" ), @@ -584,9 +578,7 @@ def test_get_unknown_viz_chart_snapshot(mocked_data_source): title="My Query Sankey", description="", lastModified=ChangeAuditStamps( - created=AuditStamp( - time=1628882009571, actor="urn:li:corpuser:unknown" - ), + created=None, lastModified=AuditStamp( time=1628882009571, actor="urn:li:corpuser:unknown" ), @@ -711,9 +703,7 @@ def test_get_chart_snapshot_parse_table_names_from_sql(mocked_data_source): title="My Query Chart", description="", lastModified=ChangeAuditStamps( - created=AuditStamp( - time=1628882022544, actor="urn:li:corpuser:unknown" - ), + created=None, lastModified=AuditStamp( time=1628882022544, actor="urn:li:corpuser:unknown" ), diff --git a/metadata-ingestion/tests/unit/test_redshift_lineage.py b/metadata-ingestion/tests/unit/test_redshift_lineage.py index 366a6009ee46a..78b7169a93f3c 100644 --- a/metadata-ingestion/tests/unit/test_redshift_lineage.py +++ b/metadata-ingestion/tests/unit/test_redshift_lineage.py @@ -241,7 +241,7 @@ def test_collapse_temp_lineage(): lineage_extractor._init_temp_table_schema( database=lineage_extractor.config.database, - temp_tables=lineage_extractor.get_temp_tables(connection=connection), + temp_tables=list(lineage_extractor.get_temp_tables(connection=connection)), ) lineage_extractor._populate_lineage_map( diff --git a/metadata-ingestion/tests/unit/test_rest_sink.py b/metadata-ingestion/tests/unit/test_rest_sink.py index 7bfa09a35951b..efa6c6678a8c7 100644 --- a/metadata-ingestion/tests/unit/test_rest_sink.py +++ b/metadata-ingestion/tests/unit/test_rest_sink.py @@ -75,7 +75,15 @@ } } }, - "systemMetadata": {}, + "systemMetadata": { + "lastObserved": 0, + "lastRunId": "no-run-id-provided", + "properties": { + "clientId": "acryl-datahub", + "clientVersion": "1!0.0.0.dev0", + }, + "runId": "no-run-id-provided", + }, }, ), ( @@ -125,7 +133,15 @@ } } }, - "systemMetadata": {}, + "systemMetadata": { + "lastObserved": 0, + "lastRunId": "no-run-id-provided", + "properties": { + "clientId": "acryl-datahub", + "clientVersion": "1!0.0.0.dev0", + }, + "runId": "no-run-id-provided", + }, }, ), ( @@ -161,7 +177,15 @@ } } }, - "systemMetadata": {}, + "systemMetadata": { + "lastObserved": 0, + "lastRunId": "no-run-id-provided", + "properties": { + "clientId": "acryl-datahub", + "clientVersion": "1!0.0.0.dev0", + }, + "runId": "no-run-id-provided", + }, }, ), ( @@ -238,6 +262,15 @@ "value": '{"owners": [{"owner": "urn:li:corpuser:fbar", "type": "DATAOWNER"}], "ownerTypes": {}, "lastModified": {"time": 0, "actor": "urn:li:corpuser:fbar"}}', "contentType": "application/json", }, + "systemMetadata": { + "lastObserved": 0, + "lastRunId": "no-run-id-provided", + "properties": { + "clientId": "acryl-datahub", + "clientVersion": "1!0.0.0.dev0", + }, + "runId": "no-run-id-provided", + }, } }, ), diff --git a/metadata-ingestion/tests/unit/test_serialized_lru_cache.py b/metadata-ingestion/tests/unit/test_serialized_lru_cache.py new file mode 100644 index 0000000000000..2b937e700b437 --- /dev/null +++ b/metadata-ingestion/tests/unit/test_serialized_lru_cache.py @@ -0,0 +1,92 @@ +import threading +import time + +from datahub.utilities.perf_timer import PerfTimer +from datahub.utilities.serialized_lru_cache import serialized_lru_cache + + +def test_cache_hit() -> None: + @serialized_lru_cache(maxsize=2) + def fetch_data(x): + return x * 2 + + assert fetch_data(1) == 2 # Cache miss + assert fetch_data(1) == 2 # Cache hit + assert fetch_data.cache_info().hits == 1 # type: ignore + assert fetch_data.cache_info().misses == 1 # type: ignore + + +def test_cache_eviction() -> None: + @serialized_lru_cache(maxsize=2) + def compute(x): + return x * 2 + + compute(1) + compute(2) + compute(3) # Should evict the first entry (1) + assert compute.cache_info().currsize == 2 # type: ignore + assert compute.cache_info().misses == 3 # type: ignore + assert compute(1) == 2 # Cache miss, since it was evicted + assert compute.cache_info().misses == 4 # type: ignore + + +def test_thread_safety() -> None: + @serialized_lru_cache(maxsize=5) + def compute(x): + time.sleep(0.2) # Simulate some delay + return x * 2 + + threads = [] + results = [None] * 10 + + def thread_func(index, arg): + results[index] = compute(arg) + + with PerfTimer() as timer: + for i in range(10): + thread = threading.Thread(target=thread_func, args=(i, i % 5)) + threads.append(thread) + thread.start() + + for thread in threads: + thread.join() + + assert len(set(results)) == 5 # Only 5 unique results should be there + assert compute.cache_info().currsize <= 5 # type: ignore + # Only 5 unique calls should miss the cache + assert compute.cache_info().misses == 5 # type: ignore + + # Should take less than 1 second. If not, it means all calls were run serially. + assert timer.elapsed_seconds() < 1 + + +def test_concurrent_access_to_same_key() -> None: + @serialized_lru_cache(maxsize=3) + def compute(x: int) -> int: + time.sleep(0.2) # Simulate some delay + return x * 2 + + threads = [] + results = [] + + def thread_func(): + results.append(compute(1)) + + with PerfTimer() as timer: + for _ in range(10): + thread = threading.Thread(target=thread_func) + threads.append(thread) + thread.start() + + for thread in threads: + thread.join() + + assert all(result == 2 for result in results) # All should compute the same result + + # 9 hits, as the first one is a miss + assert compute.cache_info().hits == 9 # type: ignore + # Only the first call is a miss + assert compute.cache_info().misses == 1 # type: ignore + + # Should take less than 1 second. If not, it means all calls were run serially. + assert timer.elapsed_seconds() < 1 diff --git a/metadata-ingestion/tests/unit/test_snowflake_source.py b/metadata-ingestion/tests/unit/test_snowflake_source.py index 69a7510692df1..3353e74449c95 100644 --- a/metadata-ingestion/tests/unit/test_snowflake_source.py +++ b/metadata-ingestion/tests/unit/test_snowflake_source.py @@ -14,7 +14,7 @@ SnowflakeCloudProvider, ) from datahub.ingestion.source.snowflake.snowflake_config import ( - DEFAULT_TABLES_DENY_LIST, + DEFAULT_TEMP_TABLES_PATTERNS, SnowflakeV2Config, ) from datahub.ingestion.source.snowflake.snowflake_query import ( @@ -24,7 +24,7 @@ from datahub.ingestion.source.snowflake.snowflake_usage_v2 import ( SnowflakeObjectAccessEntry, ) -from datahub.ingestion.source.snowflake.snowflake_utils import SnowflakeCommonMixin +from datahub.ingestion.source.snowflake.snowflake_utils import SnowsightUrlBuilder from datahub.ingestion.source.snowflake.snowflake_v2 import SnowflakeV2Source from tests.test_helpers import test_connection_helpers @@ -445,7 +445,9 @@ def test_aws_cloud_region_from_snowflake_region_id(): ( cloud, cloud_region_id, - ) = SnowflakeV2Source.get_cloud_region_from_snowflake_region_id("aws_ca_central_1") + ) = SnowsightUrlBuilder.get_cloud_region_from_snowflake_region_id( + "aws_ca_central_1" + ) assert cloud == SnowflakeCloudProvider.AWS assert cloud_region_id == "ca-central-1" @@ -453,7 +455,9 @@ def test_aws_cloud_region_from_snowflake_region_id(): ( cloud, cloud_region_id, - ) = SnowflakeV2Source.get_cloud_region_from_snowflake_region_id("aws_us_east_1_gov") + ) = SnowsightUrlBuilder.get_cloud_region_from_snowflake_region_id( + "aws_us_east_1_gov" + ) assert cloud == SnowflakeCloudProvider.AWS assert cloud_region_id == "us-east-1" @@ -463,7 +467,9 @@ def test_google_cloud_region_from_snowflake_region_id(): ( cloud, cloud_region_id, - ) = SnowflakeV2Source.get_cloud_region_from_snowflake_region_id("gcp_europe_west2") + ) = SnowsightUrlBuilder.get_cloud_region_from_snowflake_region_id( + "gcp_europe_west2" + ) assert cloud == SnowflakeCloudProvider.GCP assert cloud_region_id == "europe-west2" @@ -473,7 +479,7 @@ def test_azure_cloud_region_from_snowflake_region_id(): ( cloud, cloud_region_id, - ) = SnowflakeV2Source.get_cloud_region_from_snowflake_region_id( + ) = SnowsightUrlBuilder.get_cloud_region_from_snowflake_region_id( "azure_switzerlandnorth" ) @@ -483,7 +489,7 @@ def test_azure_cloud_region_from_snowflake_region_id(): ( cloud, cloud_region_id, - ) = SnowflakeV2Source.get_cloud_region_from_snowflake_region_id( + ) = SnowsightUrlBuilder.get_cloud_region_from_snowflake_region_id( "azure_centralindia" ) @@ -493,7 +499,7 @@ def test_azure_cloud_region_from_snowflake_region_id(): def test_unknown_cloud_region_from_snowflake_region_id(): with pytest.raises(Exception, match="Unknown snowflake region"): - SnowflakeV2Source.get_cloud_region_from_snowflake_region_id( + SnowsightUrlBuilder.get_cloud_region_from_snowflake_region_id( "somecloud_someregion" ) @@ -529,8 +535,10 @@ def test_snowflake_query_create_deny_regex_sql(): ) assert ( - create_deny_regex_sql_filter(DEFAULT_TABLES_DENY_LIST, ["upstream_table_name"]) - == r"NOT RLIKE(upstream_table_name,'.*\.FIVETRAN_.*_STAGING\..*','i') AND NOT RLIKE(upstream_table_name,'.*__DBT_TMP$','i') AND NOT RLIKE(upstream_table_name,'.*\.SEGMENT_[a-f0-9]{8}[-_][a-f0-9]{4}[-_][a-f0-9]{4}[-_][a-f0-9]{4}[-_][a-f0-9]{12}','i') AND NOT RLIKE(upstream_table_name,'.*\.STAGING_.*_[a-f0-9]{8}[-_][a-f0-9]{4}[-_][a-f0-9]{4}[-_][a-f0-9]{4}[-_][a-f0-9]{12}','i')" + create_deny_regex_sql_filter( + DEFAULT_TEMP_TABLES_PATTERNS, ["upstream_table_name"] + ) + == r"NOT RLIKE(upstream_table_name,'.*\.FIVETRAN_.*_STAGING\..*','i') AND NOT RLIKE(upstream_table_name,'.*__DBT_TMP$','i') AND NOT RLIKE(upstream_table_name,'.*\.SEGMENT_[a-f0-9]{8}[-_][a-f0-9]{4}[-_][a-f0-9]{4}[-_][a-f0-9]{4}[-_][a-f0-9]{12}','i') AND NOT RLIKE(upstream_table_name,'.*\.STAGING_.*_[a-f0-9]{8}[-_][a-f0-9]{4}[-_][a-f0-9]{4}[-_][a-f0-9]{4}[-_][a-f0-9]{12}','i') AND NOT RLIKE(upstream_table_name,'.*\.(GE_TMP_|GE_TEMP_|GX_TEMP_)[0-9A-F]{8}','i')" ) @@ -588,26 +596,15 @@ def test_email_filter_query_generation_with_case_insensitive_filter(): def test_create_snowsight_base_url_us_west(): - ( - cloud, - cloud_region_id, - ) = SnowflakeCommonMixin.get_cloud_region_from_snowflake_region_id("aws_us_west_2") - - result = SnowflakeCommonMixin.create_snowsight_base_url( - "account_locator", cloud_region_id, cloud, False - ) + result = SnowsightUrlBuilder( + "account_locator", "aws_us_west_2", privatelink=False + ).snowsight_base_url assert result == "https://app.snowflake.com/us-west-2/account_locator/" def test_create_snowsight_base_url_ap_northeast_1(): - ( - cloud, - cloud_region_id, - ) = SnowflakeCommonMixin.get_cloud_region_from_snowflake_region_id( - "aws_ap_northeast_1" - ) + result = SnowsightUrlBuilder( + "account_locator", "aws_ap_northeast_1", privatelink=False + ).snowsight_base_url - result = SnowflakeCommonMixin.create_snowsight_base_url( - "account_locator", cloud_region_id, cloud, False - ) assert result == "https://app.snowflake.com/ap-northeast-1.aws/account_locator/" diff --git a/metadata-ingestion/tests/unit/test_tableau_source.py b/metadata-ingestion/tests/unit/test_tableau_source.py index 41117914d552e..f5410b161ed70 100644 --- a/metadata-ingestion/tests/unit/test_tableau_source.py +++ b/metadata-ingestion/tests/unit/test_tableau_source.py @@ -1,12 +1,12 @@ import pytest import datahub.ingestion.source.tableau_constant as c -from datahub.ingestion.source.tableau import TableauSource +from datahub.ingestion.source.tableau import TableauSiteSource from datahub.ingestion.source.tableau_common import get_filter_pages, make_filter def test_tableau_source_unescapes_lt(): - res = TableauSource._clean_tableau_query_parameters( + res = TableauSiteSource._clean_tableau_query_parameters( "select * from t where c1 << 135" ) @@ -14,7 +14,7 @@ def test_tableau_source_unescapes_lt(): def test_tableau_source_unescapes_gt(): - res = TableauSource._clean_tableau_query_parameters( + res = TableauSiteSource._clean_tableau_query_parameters( "select * from t where c1 >> 135" ) @@ -22,7 +22,7 @@ def test_tableau_source_unescapes_gt(): def test_tableau_source_unescapes_gte(): - res = TableauSource._clean_tableau_query_parameters( + res = TableauSiteSource._clean_tableau_query_parameters( "select * from t where c1 >>= 135" ) @@ -30,7 +30,7 @@ def test_tableau_source_unescapes_gte(): def test_tableau_source_unescapeslgte(): - res = TableauSource._clean_tableau_query_parameters( + res = TableauSiteSource._clean_tableau_query_parameters( "select * from t where c1 <<= 135" ) @@ -38,7 +38,7 @@ def test_tableau_source_unescapeslgte(): def test_tableau_source_doesnt_touch_not_escaped(): - res = TableauSource._clean_tableau_query_parameters( + res = TableauSiteSource._clean_tableau_query_parameters( "select * from t where c1 < 135 and c2 > 15" ) @@ -70,7 +70,7 @@ def test_tableau_source_doesnt_touch_not_escaped(): @pytest.mark.parametrize("p", TABLEAU_PARAMS) def test_tableau_source_cleanups_tableau_parameters_in_equi_predicates(p): assert ( - TableauSource._clean_tableau_query_parameters( + TableauSiteSource._clean_tableau_query_parameters( f"select * from t where c1 = {p} and c2 = {p} and c3 = 7" ) == "select * from t where c1 = 1 and c2 = 1 and c3 = 7" @@ -80,7 +80,7 @@ def test_tableau_source_cleanups_tableau_parameters_in_equi_predicates(p): @pytest.mark.parametrize("p", TABLEAU_PARAMS) def test_tableau_source_cleanups_tableau_parameters_in_lt_gt_predicates(p): assert ( - TableauSource._clean_tableau_query_parameters( + TableauSiteSource._clean_tableau_query_parameters( f"select * from t where c1 << {p} and c2<<{p} and c3 >> {p} and c4>>{p} or {p} >> c1 and {p}>>c2 and {p} << c3 and {p}< 1 and c4>1 or 1 > c1 and 1>c2 and 1 < c3 and 1>= {p} and c4>>={p} or {p} >>= c1 and {p}>>=c2 and {p} <<= c3 and {p}<<=c4" ) == "select * from t where c1 <= 1 and c2<=1 and c3 >= 1 and c4>=1 or 1 >= c1 and 1>=c2 and 1 <= c3 and 1<=c4" @@ -100,7 +100,7 @@ def test_tableau_source_cleanups_tableau_parameters_in_lte_gte_predicates(p): @pytest.mark.parametrize("p", TABLEAU_PARAMS) def test_tableau_source_cleanups_tableau_parameters_in_join_predicate(p): assert ( - TableauSource._clean_tableau_query_parameters( + TableauSiteSource._clean_tableau_query_parameters( f"select * from t1 inner join t2 on t1.id = t2.id and t2.c21 = {p} and t1.c11 = 123 + {p}" ) == "select * from t1 inner join t2 on t1.id = t2.id and t2.c21 = 1 and t1.c11 = 123 + 1" @@ -110,7 +110,7 @@ def test_tableau_source_cleanups_tableau_parameters_in_join_predicate(p): @pytest.mark.parametrize("p", TABLEAU_PARAMS) def test_tableau_source_cleanups_tableau_parameters_in_complex_expressions(p): assert ( - TableauSource._clean_tableau_query_parameters( + TableauSiteSource._clean_tableau_query_parameters( f"select myudf1(c1, {p}, c2) / myudf2({p}) > ({p} + 3 * {p} * c5) * {p} - c4" ) == "select myudf1(c1, 1, c2) / myudf2(1) > (1 + 3 * 1 * c5) * 1 - c4" @@ -120,7 +120,7 @@ def test_tableau_source_cleanups_tableau_parameters_in_complex_expressions(p): @pytest.mark.parametrize("p", TABLEAU_PARAMS) def test_tableau_source_cleanups_tableau_parameters_in_udfs(p): assert ( - TableauSource._clean_tableau_query_parameters(f"select myudf({p}) from t") + TableauSiteSource._clean_tableau_query_parameters(f"select myudf({p}) from t") == "select myudf(1) from t" ) diff --git a/metadata-ingestion/tests/unit/test_transform_dataset.py b/metadata-ingestion/tests/unit/test_transform_dataset.py index a0deae972badb..4c4e1620f4faa 100644 --- a/metadata-ingestion/tests/unit/test_transform_dataset.py +++ b/metadata-ingestion/tests/unit/test_transform_dataset.py @@ -70,7 +70,10 @@ from datahub.ingestion.transformer.dataset_domain_based_on_tags import ( DatasetTagDomainMapper, ) -from datahub.ingestion.transformer.dataset_transformer import DatasetTransformer +from datahub.ingestion.transformer.dataset_transformer import ( + DatasetTransformer, + TagTransformer, +) from datahub.ingestion.transformer.extract_dataset_tags import ExtractDatasetTags from datahub.ingestion.transformer.extract_ownership_from_tags import ( ExtractOwnersFromTagsTransformer, @@ -86,6 +89,7 @@ SimpleRemoveDatasetOwnership, ) from datahub.ingestion.transformer.replace_external_url import ReplaceExternalUrl +from datahub.ingestion.transformer.tags_to_terms import TagsToTermMapper from datahub.metadata.schema_classes import ( BrowsePathsClass, DatasetPropertiesClass, @@ -1891,12 +1895,14 @@ def test_pattern_dataset_schema_tags_transformation(mock_time): def run_dataset_transformer_pipeline( - transformer_type: Type[DatasetTransformer], + transformer_type: Type[Union[DatasetTransformer, TagTransformer]], aspect: Optional[builder.Aspect], config: dict, - pipeline_context: PipelineContext = PipelineContext(run_id="transformer_pipe_line"), + pipeline_context: Optional[PipelineContext] = None, use_mce: bool = False, ) -> List[RecordEnvelope]: + if pipeline_context is None: + pipeline_context = PipelineContext(run_id="transformer_pipe_line") transformer: DatasetTransformer = cast( DatasetTransformer, transformer_type.create(config, pipeline_context) ) @@ -1935,7 +1941,7 @@ def test_simple_add_dataset_domain_aspect_name(mock_datahub_graph): def test_simple_add_dataset_domain(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" @@ -1944,7 +1950,7 @@ def test_simple_add_dataset_domain(mock_datahub_graph): output = run_dataset_transformer_pipeline( transformer_type=SimpleAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={"domains": [acryl_domain]}, pipeline_context=pipeline_context, ) @@ -1957,13 +1963,13 @@ def test_simple_add_dataset_domain(mock_datahub_graph): assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 2 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains def test_simple_add_dataset_domain_mce_support(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" @@ -1973,7 +1979,7 @@ def test_simple_add_dataset_domain_mce_support(mock_datahub_graph): output = run_dataset_transformer_pipeline( transformer_type=SimpleAddDatasetDomain, aspect=None, - config={"domains": [gslab_domain, acryl_domain]}, + config={"domains": [datahub_domain, acryl_domain]}, pipeline_context=pipeline_context, use_mce=True, ) @@ -1988,13 +1994,13 @@ def test_simple_add_dataset_domain_mce_support(mock_datahub_graph): assert isinstance(output[1].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[1].record.aspect) assert len(transformed_aspect.domains) == 2 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains def test_simple_add_dataset_domain_replace_existing(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" @@ -2003,7 +2009,7 @@ def test_simple_add_dataset_domain_replace_existing(mock_datahub_graph): output = run_dataset_transformer_pipeline( transformer_type=SimpleAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={"replace_existing": True, "domains": [acryl_domain]}, pipeline_context=pipeline_context, ) @@ -2016,13 +2022,13 @@ def test_simple_add_dataset_domain_replace_existing(mock_datahub_graph): assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 1 - assert gslab_domain not in transformed_aspect.domains + assert datahub_domain not in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains def test_simple_add_dataset_domain_semantics_overwrite(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") server_domain = builder.make_domain_urn("test.io") pipeline_context = PipelineContext(run_id="transformer_pipe_line") @@ -2036,7 +2042,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: output = run_dataset_transformer_pipeline( transformer_type=SimpleAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "semantics": TransformerSemantics.OVERWRITE, "domains": [acryl_domain], @@ -2052,7 +2058,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 2 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains assert server_domain not in transformed_aspect.domains @@ -2061,7 +2067,7 @@ def test_simple_add_dataset_domain_semantics_patch( pytestconfig, tmp_path, mock_time, mock_datahub_graph ): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") server_domain = builder.make_domain_urn("test.io") pipeline_context = PipelineContext(run_id="transformer_pipe_line") @@ -2075,7 +2081,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: output = run_dataset_transformer_pipeline( transformer_type=SimpleAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "replace_existing": False, "semantics": TransformerSemantics.PATCH, @@ -2092,7 +2098,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 3 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains assert server_domain in transformed_aspect.domains @@ -2111,7 +2117,7 @@ def test_pattern_add_dataset_domain_aspect_name(mock_datahub_graph): def test_pattern_add_dataset_domain_match(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*" pipeline_context: PipelineContext = PipelineContext( @@ -2121,7 +2127,7 @@ def test_pattern_add_dataset_domain_match(mock_datahub_graph): output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "domain_pattern": {"rules": {pattern: [acryl_domain]}}, }, @@ -2136,13 +2142,13 @@ def test_pattern_add_dataset_domain_match(mock_datahub_graph): assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 2 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains def test_pattern_add_dataset_domain_no_match(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:invalid,.*" pipeline_context: PipelineContext = PipelineContext( @@ -2152,7 +2158,7 @@ def test_pattern_add_dataset_domain_no_match(mock_datahub_graph): output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "domain_pattern": {"rules": {pattern: [acryl_domain]}}, }, @@ -2167,13 +2173,13 @@ def test_pattern_add_dataset_domain_no_match(mock_datahub_graph): assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 1 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain not in transformed_aspect.domains def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*" pipeline_context: PipelineContext = PipelineContext( @@ -2183,7 +2189,7 @@ def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph): output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "replace_existing": True, "domain_pattern": {"rules": {pattern: [acryl_domain]}}, @@ -2199,13 +2205,13 @@ def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph): assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 1 - assert gslab_domain not in transformed_aspect.domains + assert datahub_domain not in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains def test_pattern_add_dataset_domain_replace_existing_no_match(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:invalid,.*" pipeline_context: PipelineContext = PipelineContext( @@ -2215,7 +2221,7 @@ def test_pattern_add_dataset_domain_replace_existing_no_match(mock_datahub_graph output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "replace_existing": True, "domain_pattern": {"rules": {pattern: [acryl_domain]}}, @@ -2235,7 +2241,7 @@ def test_pattern_add_dataset_domain_replace_existing_no_match(mock_datahub_graph def test_pattern_add_dataset_domain_semantics_overwrite(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") server_domain = builder.make_domain_urn("test.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*" @@ -2250,7 +2256,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "semantics": TransformerSemantics.OVERWRITE, "domain_pattern": {"rules": {pattern: [acryl_domain]}}, @@ -2266,7 +2272,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 2 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains assert server_domain not in transformed_aspect.domains @@ -2275,7 +2281,7 @@ def test_pattern_add_dataset_domain_semantics_patch( pytestconfig, tmp_path, mock_time, mock_datahub_graph ): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") server_domain = builder.make_domain_urn("test.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*" @@ -2290,7 +2296,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "replace_existing": False, "semantics": TransformerSemantics.PATCH, @@ -2307,7 +2313,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 3 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains assert server_domain in transformed_aspect.domains @@ -3651,3 +3657,213 @@ def fake_get_tags(entity_urn: str) -> Optional[models.GlobalTagsClass]: assert len(transformed_aspect.domains) == 1 assert acryl_domain in transformed_aspect.domains assert server_domain not in transformed_aspect.domains + + +def test_tags_to_terms_transformation(mock_datahub_graph): + # Create domain URNs for the test + term_urn_example1 = builder.make_term_urn("example1") + term_urn_example2 = builder.make_term_urn("example2") + + def fake_get_tags(entity_urn: str) -> models.GlobalTagsClass: + return models.GlobalTagsClass( + tags=[ + TagAssociationClass(tag=builder.make_tag_urn("example1")), + TagAssociationClass(tag=builder.make_tag_urn("example2")), + ] + ) + + # fake the server response + def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass: + return models.SchemaMetadataClass( + schemaName="customer", # not used + platform=builder.make_data_platform_urn( + "hive" + ), # important <- platform must be an urn + version=0, + # when the source system has a notion of versioning of schemas, insert this in, otherwise leave as 0 + hash="", + # when the source system has a notion of unique schemas identified via hash, include a hash, else leave it as empty string + platformSchema=models.OtherSchemaClass( + rawSchema="__insert raw schema here__" + ), + fields=[ + models.SchemaFieldClass( + fieldPath="first_name", + globalTags=models.GlobalTagsClass( + tags=[ + models.TagAssociationClass( + tag=builder.make_tag_urn("example2") + ) + ], + ), + glossaryTerms=models.GlossaryTermsClass( + terms=[ + models.GlossaryTermAssociationClass( + urn=builder.make_term_urn("pii") + ) + ], + auditStamp=models.AuditStampClass._construct_with_defaults(), + ), + type=models.SchemaFieldDataTypeClass(type=models.StringTypeClass()), + nativeDataType="VARCHAR(100)", + # use this to provide the type of the field in the source system's vernacular + ), + models.SchemaFieldClass( + fieldPath="mobile_number", + glossaryTerms=models.GlossaryTermsClass( + terms=[ + models.GlossaryTermAssociationClass( + urn=builder.make_term_urn("pii") + ) + ], + auditStamp=models.AuditStampClass._construct_with_defaults(), + ), + type=models.SchemaFieldDataTypeClass(type=models.StringTypeClass()), + nativeDataType="VARCHAR(100)", + # use this to provide the type of the field in the source system's vernacular + ), + ], + ) + + pipeline_context = PipelineContext(run_id="transformer_pipe_line") + pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph.get_tags = fake_get_tags # type: ignore + pipeline_context.graph.get_schema_metadata = fake_schema_metadata # type: ignore + + # Configuring the transformer + config = {"tags": ["example1", "example2"]} + + # Running the transformer within a test pipeline + output = run_dataset_transformer_pipeline( + transformer_type=TagsToTermMapper, + aspect=models.GlossaryTermsClass( + terms=[ + models.GlossaryTermAssociationClass(urn=builder.make_term_urn("pii")) + ], + auditStamp=models.AuditStampClass._construct_with_defaults(), + ), + config=config, + pipeline_context=pipeline_context, + ) + + # Expected results + expected_terms = [term_urn_example2, term_urn_example1] + + # Verify the output + assert len(output) == 2 # One for result and one for end of stream + terms_aspect = output[0].record.aspect + assert isinstance(terms_aspect, models.GlossaryTermsClass) + assert len(terms_aspect.terms) == len(expected_terms) + assert set(term.urn for term in terms_aspect.terms) == { + "urn:li:glossaryTerm:example1", + "urn:li:glossaryTerm:example2", + } + + +def test_tags_to_terms_with_no_matching_terms(mock_datahub_graph): + # Setup for test where no tags match the provided term mappings + def fake_get_tags_no_match(entity_urn: str) -> models.GlobalTagsClass: + return models.GlobalTagsClass( + tags=[ + TagAssociationClass(tag=builder.make_tag_urn("nonMatchingTag1")), + TagAssociationClass(tag=builder.make_tag_urn("nonMatchingTag2")), + ] + ) + + pipeline_context = PipelineContext(run_id="transformer_pipe_line") + pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph.get_tags = fake_get_tags_no_match # type: ignore + + # No matching terms in config + config = {"tags": ["example1", "example2"]} + + # Running the transformer within a test pipeline + output = run_dataset_transformer_pipeline( + transformer_type=TagsToTermMapper, + aspect=models.GlossaryTermsClass( + terms=[ + models.GlossaryTermAssociationClass(urn=builder.make_term_urn("pii")) + ], + auditStamp=models.AuditStampClass._construct_with_defaults(), + ), + config=config, + pipeline_context=pipeline_context, + ) + + # Verify the output + assert len(output) == 2 # One for result and one for end of stream + terms_aspect = output[0].record.aspect + assert isinstance(terms_aspect, models.GlossaryTermsClass) + assert len(terms_aspect.terms) == 1 + + +def test_tags_to_terms_with_missing_tags(mock_datahub_graph): + # Setup for test where no tags are present + def fake_get_no_tags(entity_urn: str) -> models.GlobalTagsClass: + return models.GlobalTagsClass(tags=[]) + + pipeline_context = PipelineContext(run_id="transformer_pipe_line") + pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph.get_tags = fake_get_no_tags # type: ignore + + config = {"tags": ["example1", "example2"]} + + # Running the transformer with no tags + output = run_dataset_transformer_pipeline( + transformer_type=TagsToTermMapper, + aspect=models.GlossaryTermsClass( + terms=[ + models.GlossaryTermAssociationClass(urn=builder.make_term_urn("pii")) + ], + auditStamp=models.AuditStampClass._construct_with_defaults(), + ), + config=config, + pipeline_context=pipeline_context, + ) + + # Verify that no terms are added when there are no tags + assert len(output) == 2 + terms_aspect = output[0].record.aspect + assert isinstance(terms_aspect, models.GlossaryTermsClass) + assert len(terms_aspect.terms) == 1 + + +def test_tags_to_terms_with_partial_match(mock_datahub_graph): + # Setup for partial match scenario + def fake_get_partial_match_tags(entity_urn: str) -> models.GlobalTagsClass: + return models.GlobalTagsClass( + tags=[ + TagAssociationClass( + tag=builder.make_tag_urn("example1") + ), # Should match + TagAssociationClass( + tag=builder.make_tag_urn("nonMatchingTag") + ), # No match + ] + ) + + pipeline_context = PipelineContext(run_id="transformer_pipe_line") + pipeline_context.graph = mock_datahub_graph(DatahubClientConfig()) + pipeline_context.graph.get_tags = fake_get_partial_match_tags # type: ignore + + config = {"tags": ["example1"]} # Only 'example1' has a term mapped + + # Running the transformer with partial matching tags + output = run_dataset_transformer_pipeline( + transformer_type=TagsToTermMapper, + aspect=models.GlossaryTermsClass( + terms=[ + models.GlossaryTermAssociationClass(urn=builder.make_term_urn("pii")) + ], + auditStamp=models.AuditStampClass._construct_with_defaults(), + ), + config=config, + pipeline_context=pipeline_context, + ) + + # Verify that only matched term is added + assert len(output) == 2 + terms_aspect = output[0].record.aspect + assert isinstance(terms_aspect, models.GlossaryTermsClass) + assert len(terms_aspect.terms) == 1 + assert terms_aspect.terms[0].urn == "urn:li:glossaryTerm:example1" diff --git a/metadata-ingestion/tests/unit/test_usage_common.py b/metadata-ingestion/tests/unit/test_usage_common.py index 1e2b2b6999177..e01f0ea77df83 100644 --- a/metadata-ingestion/tests/unit/test_usage_common.py +++ b/metadata-ingestion/tests/unit/test_usage_common.py @@ -190,8 +190,9 @@ def test_make_usage_workunit(): assert ( wu.id == f"{_simple_urn_builder(resource)}-{USAGE_ASPECT_NAME}-{ts_timestamp}" ) - assert isinstance(wu.get_metadata()["metadata"], MetadataChangeProposalWrapper) - du: DatasetUsageStatisticsClass = wu.get_metadata()["metadata"].aspect + assert isinstance(wu.metadata, MetadataChangeProposalWrapper) + assert isinstance(wu.metadata.aspect, DatasetUsageStatisticsClass) + du: DatasetUsageStatisticsClass = wu.metadata.aspect assert du.totalSqlQueries == 1 assert du.topSqlQueries assert du.topSqlQueries.pop() == test_query @@ -225,8 +226,9 @@ def test_query_formatting(): assert ( wu.id == f"{_simple_urn_builder(resource)}-{USAGE_ASPECT_NAME}-{ts_timestamp}" ) - assert isinstance(wu.get_metadata()["metadata"], MetadataChangeProposalWrapper) - du: DatasetUsageStatisticsClass = wu.get_metadata()["metadata"].aspect + assert isinstance(wu.metadata, MetadataChangeProposalWrapper) + assert isinstance(wu.metadata.aspect, DatasetUsageStatisticsClass) + du: DatasetUsageStatisticsClass = wu.metadata.aspect assert du.totalSqlQueries == 1 assert du.topSqlQueries assert du.topSqlQueries.pop() == formatted_test_query @@ -260,8 +262,9 @@ def test_query_trimming(): assert ( wu.id == f"{_simple_urn_builder(resource)}-{USAGE_ASPECT_NAME}-{ts_timestamp}" ) - assert isinstance(wu.get_metadata()["metadata"], MetadataChangeProposalWrapper) - du: DatasetUsageStatisticsClass = wu.get_metadata()["metadata"].aspect + assert isinstance(wu.metadata, MetadataChangeProposalWrapper) + assert isinstance(wu.metadata.aspect, DatasetUsageStatisticsClass) + du: DatasetUsageStatisticsClass = wu.metadata.aspect assert du.totalSqlQueries == 1 assert du.topSqlQueries assert du.topSqlQueries.pop() == "select * from te ..." @@ -299,8 +302,9 @@ def test_make_usage_workunit_include_top_n_queries(): assert ( wu.id == f"{_simple_urn_builder(resource)}-{USAGE_ASPECT_NAME}-{ts_timestamp}" ) - assert isinstance(wu.get_metadata()["metadata"], MetadataChangeProposalWrapper) - du: DatasetUsageStatisticsClass = wu.get_metadata()["metadata"].aspect + assert isinstance(wu.metadata, MetadataChangeProposalWrapper) + assert isinstance(wu.metadata.aspect, DatasetUsageStatisticsClass) + du: DatasetUsageStatisticsClass = wu.metadata.aspect assert du.totalSqlQueries == 1 assert du.topSqlQueries is None diff --git a/metadata-ingestion/tests/unit/utilities/test_advanced_thread_executor.py b/metadata-ingestion/tests/unit/utilities/test_advanced_thread_executor.py deleted file mode 100644 index 7b51c18a85c5f..0000000000000 --- a/metadata-ingestion/tests/unit/utilities/test_advanced_thread_executor.py +++ /dev/null @@ -1,128 +0,0 @@ -import time -from concurrent.futures import Future - -from datahub.utilities.advanced_thread_executor import ( - BackpressureAwareExecutor, - PartitionExecutor, -) -from datahub.utilities.perf_timer import PerfTimer - - -def test_partitioned_executor(): - executing_tasks = set() - done_tasks = set() - - def task(key: str, id: str) -> None: - executing_tasks.add((key, id)) - time.sleep(0.8) - done_tasks.add(id) - executing_tasks.remove((key, id)) - - with PartitionExecutor(max_workers=2, max_pending=10) as executor: - # Submit tasks with the same key. They should be executed sequentially. - executor.submit("key1", task, "key1", "task1") - executor.submit("key1", task, "key1", "task2") - executor.submit("key1", task, "key1", "task3") - - # Submit a task with a different key. It should be executed in parallel. - executor.submit("key2", task, "key2", "task4") - - saw_keys_in_parallel = False - while executing_tasks or not done_tasks: - keys_executing = [key for key, _ in executing_tasks] - assert list(sorted(keys_executing)) == list( - sorted(set(keys_executing)) - ), "partitioning not working" - - if len(keys_executing) == 2: - saw_keys_in_parallel = True - - time.sleep(0.1) - - executor.flush() - assert saw_keys_in_parallel - assert not executing_tasks - assert done_tasks == {"task1", "task2", "task3", "task4"} - - -def test_partitioned_executor_bounding(): - task_duration = 0.5 - done_tasks = set() - - def on_done(future: Future) -> None: - done_tasks.add(future.result()) - - def task(id: str) -> str: - time.sleep(task_duration) - return id - - with PartitionExecutor( - max_workers=5, max_pending=10 - ) as executor, PerfTimer() as timer: - # The first 15 submits should be non-blocking. - for i in range(15): - executor.submit(f"key{i}", task, f"task{i}", done_callback=on_done) - assert timer.elapsed_seconds() < task_duration - - # This submit should block. - executor.submit("key-blocking", task, "task-blocking", done_callback=on_done) - assert timer.elapsed_seconds() > task_duration - - # Wait for everything to finish. - executor.flush() - assert len(done_tasks) == 16 - - -def test_backpressure_aware_executor_simple(): - def task(i): - return i - - assert { - res.result() - for res in BackpressureAwareExecutor.map( - task, ((i,) for i in range(10)), max_workers=2 - ) - } == set(range(10)) - - -def test_backpressure_aware_executor_advanced(): - task_duration = 0.5 - started = set() - executed = set() - - def task(x, y): - assert x + 1 == y - started.add(x) - time.sleep(task_duration) - executed.add(x) - return x - - args_list = [(i, i + 1) for i in range(10)] - - with PerfTimer() as timer: - results = BackpressureAwareExecutor.map( - task, args_list, max_workers=2, max_pending=4 - ) - assert timer.elapsed_seconds() < task_duration - - # No tasks should have completed yet. - assert len(executed) == 0 - - # Consume the first result. - first_result = next(results) - assert 0 <= first_result.result() < 4 - assert timer.elapsed_seconds() > task_duration - - # By now, the first four tasks should have started. - time.sleep(task_duration) - assert {0, 1, 2, 3}.issubset(started) - assert 2 <= len(executed) <= 4 - - # Finally, consume the rest of the results. - assert {r.result() for r in results} == { - i for i in range(10) if i != first_result.result() - } - - # Validate that the entire process took about 5-10x the task duration. - # That's because we have 2 workers and 10 tasks. - assert 5 * task_duration < timer.elapsed_seconds() < 10 * task_duration diff --git a/metadata-ingestion/tests/unit/utilities/test_backpressure_aware_executor.py b/metadata-ingestion/tests/unit/utilities/test_backpressure_aware_executor.py new file mode 100644 index 0000000000000..5b320b8a23254 --- /dev/null +++ b/metadata-ingestion/tests/unit/utilities/test_backpressure_aware_executor.py @@ -0,0 +1,59 @@ +import time + +from datahub.utilities.backpressure_aware_executor import BackpressureAwareExecutor +from datahub.utilities.perf_timer import PerfTimer + + +def test_backpressure_aware_executor_simple(): + def task(i): + return i + + assert { + res.result() + for res in BackpressureAwareExecutor.map( + task, ((i,) for i in range(10)), max_workers=2 + ) + } == set(range(10)) + + +def test_backpressure_aware_executor_advanced(): + task_duration = 0.5 + started = set() + executed = set() + + def task(x, y): + assert x + 1 == y + started.add(x) + time.sleep(task_duration) + executed.add(x) + return x + + args_list = [(i, i + 1) for i in range(10)] + + with PerfTimer() as timer: + results = BackpressureAwareExecutor.map( + task, args_list, max_workers=2, max_pending=4 + ) + assert timer.elapsed_seconds() < task_duration + + # No tasks should have completed yet. + assert len(executed) == 0 + + # Consume the first result. + first_result = next(results) + assert 0 <= first_result.result() < 4 + assert timer.elapsed_seconds() > task_duration + + # By now, the first four tasks should have started. + time.sleep(task_duration) + assert {0, 1, 2, 3}.issubset(started) + assert 2 <= len(executed) <= 4 + + # Finally, consume the rest of the results. + assert {r.result() for r in results} == { + i for i in range(10) if i != first_result.result() + } + + # Validate that the entire process took about 5-10x the task duration. + # That's because we have 2 workers and 10 tasks. + assert 5 * task_duration < timer.elapsed_seconds() < 10 * task_duration diff --git a/metadata-ingestion/tests/unit/utilities/test_partition_executor.py b/metadata-ingestion/tests/unit/utilities/test_partition_executor.py new file mode 100644 index 0000000000000..81c5b898caf2b --- /dev/null +++ b/metadata-ingestion/tests/unit/utilities/test_partition_executor.py @@ -0,0 +1,150 @@ +import logging +import time +from concurrent.futures import Future + +from datahub.utilities.partition_executor import ( + BatchPartitionExecutor, + PartitionExecutor, +) +from datahub.utilities.perf_timer import PerfTimer + +logger = logging.getLogger(__name__) + + +def test_partitioned_executor(): + executing_tasks = set() + done_tasks = set() + + def task(key: str, id: str) -> None: + executing_tasks.add((key, id)) + time.sleep(0.8) + done_tasks.add(id) + executing_tasks.remove((key, id)) + + with PartitionExecutor(max_workers=2, max_pending=10) as executor: + # Submit tasks with the same key. They should be executed sequentially. + executor.submit("key1", task, "key1", "task1") + executor.submit("key1", task, "key1", "task2") + executor.submit("key1", task, "key1", "task3") + + # Submit a task with a different key. It should be executed in parallel. + executor.submit("key2", task, "key2", "task4") + + saw_keys_in_parallel = False + while executing_tasks or not done_tasks: + keys_executing = [key for key, _ in executing_tasks] + assert list(sorted(keys_executing)) == list( + sorted(set(keys_executing)) + ), "partitioning not working" + + if len(keys_executing) == 2: + saw_keys_in_parallel = True + + time.sleep(0.1) + + executor.flush() + assert saw_keys_in_parallel + assert not executing_tasks + assert done_tasks == {"task1", "task2", "task3", "task4"} + + +def test_partitioned_executor_bounding(): + task_duration = 0.5 + done_tasks = set() + + def on_done(future: Future) -> None: + done_tasks.add(future.result()) + + def task(id: str) -> str: + time.sleep(task_duration) + return id + + with PartitionExecutor( + max_workers=5, max_pending=10 + ) as executor, PerfTimer() as timer: + # The first 15 submits should be non-blocking. + for i in range(15): + executor.submit(f"key{i}", task, f"task{i}", done_callback=on_done) + assert timer.elapsed_seconds() < task_duration + + # This submit should block. + executor.submit("key-blocking", task, "task-blocking", done_callback=on_done) + assert timer.elapsed_seconds() > task_duration + + # Wait for everything to finish. + executor.flush() + assert len(done_tasks) == 16 + + +def test_batch_partition_executor_sequential_key_execution(): + executing_tasks = set() + done_tasks = set() + done_task_batches = set() + + def process_batch(batch): + for key, id in batch: + assert (key, id) not in executing_tasks, "Task is already executing" + executing_tasks.add((key, id)) + + time.sleep(0.5) # Simulate work + + for key, id in batch: + executing_tasks.remove((key, id)) + done_tasks.add(id) + + done_task_batches.add(tuple(id for _, id in batch)) + + with BatchPartitionExecutor( + max_workers=2, + max_pending=10, + max_per_batch=2, + process_batch=process_batch, + ) as executor: + # Submit tasks with the same key. The first two should get batched together. + executor.submit("key1", "key1", "task1") + executor.submit("key1", "key1", "task2") + executor.submit("key1", "key1", "task3") + + # Submit tasks with a different key. These should get their own batch. + executor.submit("key2", "key2", "task4") + executor.submit("key2", "key2", "task5") + + # Test idempotency of shutdown(). + executor.shutdown() + + # Check if all tasks were executed and completed. + assert done_tasks == { + "task1", + "task2", + "task3", + "task4", + "task5", + }, "Not all tasks completed" + + # Check the batching configuration. + assert done_task_batches == { + ("task1", "task2"), + ("task4", "task5"), + ("task3",), + } + + +def test_batch_partition_executor_max_batch_size(): + batches_processed = [] + + def process_batch(batch): + batches_processed.append(batch) + time.sleep(0.1) # Simulate batch processing time + + with BatchPartitionExecutor( + max_workers=5, max_pending=20, process_batch=process_batch, max_per_batch=2 + ) as executor: + # Submit more tasks than the max_per_batch to test batching limits. + for i in range(5): + executor.submit("key3", "key3", f"task{i}") + + # Check the batches. + logger.info(f"batches_processed: {batches_processed}") + assert len(batches_processed) == 3 + for batch in batches_processed: + assert len(batch) <= 2, "Batch size exceeded max_per_batch limit" diff --git a/metadata-ingestion/tests/unit/utilities/test_prefix_patch_builder.py b/metadata-ingestion/tests/unit/utilities/test_prefix_patch_builder.py new file mode 100644 index 0000000000000..c62bac24a0a42 --- /dev/null +++ b/metadata-ingestion/tests/unit/utilities/test_prefix_patch_builder.py @@ -0,0 +1,45 @@ +from datahub.utilities.prefix_batch_builder import PrefixGroup, build_prefix_batches + + +def test_build_prefix_batches_empty_input(): + assert build_prefix_batches([], 10, 5) == [[PrefixGroup(prefix="", names=[])]] + + +def test_build_prefix_batches_single_group(): + names = ["apple", "applet", "application"] + expected = [[PrefixGroup(prefix="", names=names)]] + assert build_prefix_batches(names, 10, 5) == expected + + +def test_build_prefix_batches_multiple_groups(): + names = ["apple", "applet", "banana", "band", "bandana"] + expected = [ + [PrefixGroup(prefix="a", names=["apple", "applet"])], + [PrefixGroup(prefix="b", names=["banana", "band", "bandana"])], + ] + assert build_prefix_batches(names, 4, 5) == expected + + +def test_build_prefix_batches_exceeds_max_batch_size(): + names = [ + "app", + "apple", + "applet", + "application", + "banana", + "band", + "bandana", + "candy", + "candle", + "dog", + ] + expected = [ + [PrefixGroup(prefix="app", names=["app"], exact_match=True)], + [PrefixGroup(prefix="appl", names=["apple", "applet", "application"])], + [PrefixGroup(prefix="b", names=["banana", "band", "bandana"])], + [ + PrefixGroup(prefix="c", names=["candle", "candy"]), + PrefixGroup(prefix="d", names=["dog"]), + ], + ] + assert build_prefix_batches(names, 3, 2) == expected diff --git a/metadata-integration/java/as-a-library.md b/metadata-integration/java/as-a-library.md index 59ee6595132fa..e38cfef080f5f 100644 --- a/metadata-integration/java/as-a-library.md +++ b/metadata-integration/java/as-a-library.md @@ -169,7 +169,7 @@ If you're interested in looking at the Kafka emitter code, it is available [here ## File Emitter -The File emitter writes metadata change proposal events (MCPs) into a JSON file that can be later handed off to the Python [File source](docs/generated/ingestion/sources/file.md) for ingestion. This works analogous to the [File sink](../../metadata-ingestion/sink_docs/file.md) in Python. This mechanism can be used when the system producing metadata events doesn't have direct connection to DataHub's REST server or Kafka brokers. The generated JSON file can be transferred later and then ingested into DataHub using the [File source](docs/generated/ingestion/sources/file.md). +The File emitter writes metadata change proposal events (MCPs) into a JSON file that can be later handed off to the Python [Metadata File source](docs/generated/ingestion/sources/metadata-file.md) for ingestion. This works analogous to the [Metadata File sink](../../metadata-ingestion/sink_docs/metadata-file.md) in Python. This mechanism can be used when the system producing metadata events doesn't have direct connection to DataHub's REST server or Kafka brokers. The generated JSON file can be transferred later and then ingested into DataHub using the [Metadata File source](docs/generated/ingestion/sources/metadata-file.md). ### Usage @@ -223,5 +223,3 @@ The File emitter only supports writing to the local filesystem currently. If you Emitter API-s are also supported for: - [Python](../../metadata-ingestion/as-a-library.md) - - diff --git a/metadata-integration/java/custom-plugin-lib/build.gradle b/metadata-integration/java/custom-plugin-lib/build.gradle index 08bbe587b49c0..305dbe9578fa0 100644 --- a/metadata-integration/java/custom-plugin-lib/build.gradle +++ b/metadata-integration/java/custom-plugin-lib/build.gradle @@ -38,6 +38,16 @@ configurations.all { exclude group: 'org.antlr' } +task sourcesJar(type: Jar) { + archiveClassifier = 'sources' + from sourceSets.main.allSource +} + +task javadocJar(type: Jar) { + archiveClassifier = 'javadoc' + from javadoc +} + shadowJar { zip64 = true archiveClassifier = '' @@ -59,7 +69,7 @@ publishing { artifactId = 'datahub-custom-plugin-lib' description = 'DataHub Java Custom Plugin dependencies' url = 'https://datahubproject.io' - artifacts = [shadowJar] + artifacts = [shadowJar, javadocJar, sourcesJar] scm { connection = 'scm:git:git://github.com/datahub-project/datahub.git' diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index 53c2a33828907..3f54497919062 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -120,6 +120,7 @@ shadowJar { relocate 'nonapi.io.github.classgraph', 'datahub.shaded.nonapi.io.github.classgraph' relocate 'org.eclipse.parsson', 'datahub.shaded.parsson' relocate 'jakarta.json', 'datahub.shaded.json' + relocate 'io.netty', 'datahub.shaded.io.netty' finalizedBy checkShadowJar } diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java index ed4cee060bd69..dd6a7ba98c87d 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java @@ -35,6 +35,7 @@ import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder; import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.config.TlsConfig; import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; @@ -45,6 +46,7 @@ import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.http2.HttpVersionPolicy; import org.apache.hc.core5.ssl.SSLContexts; import org.apache.hc.core5.util.TimeValue; @@ -106,6 +108,14 @@ public RestEmitter(RestEmitterConfig config) { config.getTimeoutSec() * 1000, java.util.concurrent.TimeUnit.MILLISECONDS) .build()); } + PoolingAsyncClientConnectionManagerBuilder poolingAsyncClientConnectionManagerBuilder = + PoolingAsyncClientConnectionManagerBuilder.create(); + + // Forcing http 1.x as 2.0 is not supported yet + TlsConfig tlsHttp1Config = + TlsConfig.copy(TlsConfig.DEFAULT).setVersionPolicy(HttpVersionPolicy.FORCE_HTTP_1).build(); + poolingAsyncClientConnectionManagerBuilder.setDefaultTlsConfig(tlsHttp1Config); + if (config.isDisableSslVerification()) { try { SSLContext sslcontext = @@ -115,15 +125,12 @@ public RestEmitter(RestEmitterConfig config) { .setSslContext(sslcontext) .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) .build(); - - httpClientBuilder.setConnectionManager( - PoolingAsyncClientConnectionManagerBuilder.create() - .setTlsStrategy(tlsStrategy) - .build()); + poolingAsyncClientConnectionManagerBuilder.setTlsStrategy(tlsStrategy); } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) { throw new RuntimeException("Error while creating insecure http client", e); } } + httpClientBuilder.setConnectionManager(poolingAsyncClientConnectionManagerBuilder.build()); httpClientBuilder.setRetryStrategy( new DatahubHttpRequestRetryStrategy( diff --git a/metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc b/metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc index 12ace42f3af27..126d5cf6bcee3 100644 --- a/metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc +++ b/metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc @@ -167,9 +167,22 @@ } ], "doc" : "Additional properties", "default" : null + }, { + "name" : "version", + "type" : [ "null", "string" ], + "doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.", + "default" : null } ] } ], - "doc" : "A string->string map of custom properties that one might want to attach to an event", + "doc" : "System properties that one might want to attach to an event", + "default" : null + }, { + "name" : "headers", + "type" : [ "null", { + "type" : "map", + "values" : "string" + } ], + "doc" : "Headers - intended to mimic http headers", "default" : null } ] } \ No newline at end of file diff --git a/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java b/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java index 46d1481836fe7..95d2060079780 100644 --- a/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java +++ b/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java @@ -17,7 +17,12 @@ import com.linkedin.common.urn.TagUrn; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringArray; +import com.linkedin.data.template.StringArrayMap; import com.linkedin.dataset.DatasetLineageType; +import com.linkedin.form.FormPrompt; +import com.linkedin.form.FormPromptType; +import com.linkedin.form.StructuredPropertyParams; import com.linkedin.metadata.aspect.patch.builder.ChartInfoPatchBuilder; import com.linkedin.metadata.aspect.patch.builder.DashboardInfoPatchBuilder; import com.linkedin.metadata.aspect.patch.builder.DataFlowInfoPatchBuilder; @@ -25,10 +30,16 @@ import com.linkedin.metadata.aspect.patch.builder.DataJobInputOutputPatchBuilder; import com.linkedin.metadata.aspect.patch.builder.DatasetPropertiesPatchBuilder; import com.linkedin.metadata.aspect.patch.builder.EditableSchemaMetadataPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.FormInfoPatchBuilder; import com.linkedin.metadata.aspect.patch.builder.OwnershipPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.StructuredPropertiesPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.StructuredPropertyDefinitionPatchBuilder; import com.linkedin.metadata.aspect.patch.builder.UpstreamLineagePatchBuilder; import com.linkedin.metadata.graph.LineageDirection; import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.structured.PrimitivePropertyValue; +import com.linkedin.structured.PropertyCardinality; +import com.linkedin.structured.PropertyValue; import datahub.client.MetadataWriteResponse; import datahub.client.file.FileEmitter; import datahub.client.file.FileEmitterConfig; @@ -37,6 +48,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.util.Arrays; +import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import org.junit.Ignore; @@ -641,4 +653,138 @@ public void testLocalDashboardInfoRemove() { System.out.println(Arrays.asList(e.getStackTrace())); } } + + @Test + @Ignore + public void testLocalStructuredPropertyDefinitionAdd() { + RestEmitter restEmitter = new RestEmitter(RestEmitterConfig.builder().build()); + try { + StringArrayMap typeQualifier = new StringArrayMap(); + typeQualifier.put( + "allowedTypes", + new StringArray( + "urn:li:entityType:datahub.corpuser", "urn:li:entityType:datahub.corpGroup")); + PropertyValue propertyValue1 = new PropertyValue(); + PrimitivePropertyValue value1 = new PrimitivePropertyValue(); + value1.setString("test value 1"); + propertyValue1.setValue(value1); + PropertyValue propertyValue2 = new PropertyValue(); + PrimitivePropertyValue value2 = new PrimitivePropertyValue(); + value2.setString("test value 2"); + propertyValue2.setValue(value2); + + MetadataChangeProposal structuredPropertyDefinitionPatch = + new StructuredPropertyDefinitionPatchBuilder() + .urn(UrnUtils.getUrn("urn:li:structuredProperty:123456")) + .setQualifiedName("test.testing.123") + .setDisplayName("Test Display Name") + .setValueType("urn:li:dataType:datahub.urn") + .setTypeQualifier(typeQualifier) + .addAllowedValue(propertyValue1) + .addAllowedValue(propertyValue2) + .setCardinality(PropertyCardinality.MULTIPLE) + .addEntityType("urn:li:entityType:datahub.dataFlow") + .setDescription("test description") + .setImmutable(true) + .build(); + + Future response = restEmitter.emit(structuredPropertyDefinitionPatch); + + System.out.println(response.get().getResponseContent()); + + } catch (IOException | ExecutionException | InterruptedException e) { + System.out.println(Arrays.asList(e.getStackTrace())); + } + } + + @Test + @Ignore + public void testLocalFormInfoAdd() { + RestEmitter restEmitter = new RestEmitter(RestEmitterConfig.builder().build()); + try { + FormPrompt newPrompt = + new FormPrompt() + .setId("1234") + .setTitle("First Prompt") + .setType(FormPromptType.STRUCTURED_PROPERTY) + .setRequired(true) + .setStructuredPropertyParams( + new StructuredPropertyParams() + .setUrn(UrnUtils.getUrn("urn:li:structuredProperty:property1"))); + FormPrompt newPrompt2 = + new FormPrompt() + .setId("abcd") + .setTitle("Second Prompt") + .setType(FormPromptType.FIELDS_STRUCTURED_PROPERTY) + .setRequired(false) + .setStructuredPropertyParams( + new StructuredPropertyParams() + .setUrn(UrnUtils.getUrn("urn:li:structuredProperty:property1"))); + + MetadataChangeProposal formInfoPatch = + new FormInfoPatchBuilder() + .urn(UrnUtils.getUrn("urn:li:form:123456")) + .addPrompts(List.of(newPrompt, newPrompt2)) + .setName("Metadata Initiative 2024 (edited)") + .setDescription("Edited description") + .setOwnershipForm(true) + .addAssignedUser("urn:li:corpuser:admin") + .addAssignedGroup("urn:li:corpGroup:jdoe") + .build(); + Future response = restEmitter.emit(formInfoPatch); + + System.out.println(response.get().getResponseContent()); + + } catch (IOException | ExecutionException | InterruptedException e) { + System.out.println(Arrays.asList(e.getStackTrace())); + } + } + + @Test + @Ignore + public void testLocalStructuredPropertiesUpdate() { + try { + MetadataChangeProposal mcp = + new StructuredPropertiesPatchBuilder() + .urn( + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)")) + .setNumberProperty( + UrnUtils.getUrn( + "urn:li:structuredProperty:io.acryl.dataManagement.replicationSLA"), + 3456) + .build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + Future response = emitter.emit(mcp, null); + System.out.println(response.get().getResponseContent()); + + } catch (IOException | ExecutionException | InterruptedException e) { + System.out.println(Arrays.asList(e.getStackTrace())); + } + } + + @Test + @Ignore + public void testLocalFormInfoRemove() { + RestEmitter restEmitter = new RestEmitter(RestEmitterConfig.builder().build()); + try { + MetadataChangeProposal formInfoPatch = + new FormInfoPatchBuilder() + .urn(UrnUtils.getUrn("urn:li:form:123456")) + .removePrompts(List.of("1234", "abcd")) + .setName("Metadata Initiative 2024 (edited - again)") + .setDescription(null) + .removeAssignedUser("urn:li:corpuser:admin") + .removeAssignedGroup("urn:li:corpGroup:jdoe") + .build(); + Future response = restEmitter.emit(formInfoPatch); + + System.out.println(response.get().getResponseContent()); + + } catch (IOException | ExecutionException | InterruptedException e) { + System.out.println(Arrays.asList(e.getStackTrace())); + } + } } diff --git a/metadata-integration/java/datahub-event/src/main/resources/MetadataChangeProposal.avsc b/metadata-integration/java/datahub-event/src/main/resources/MetadataChangeProposal.avsc index 12ace42f3af27..126d5cf6bcee3 100644 --- a/metadata-integration/java/datahub-event/src/main/resources/MetadataChangeProposal.avsc +++ b/metadata-integration/java/datahub-event/src/main/resources/MetadataChangeProposal.avsc @@ -167,9 +167,22 @@ } ], "doc" : "Additional properties", "default" : null + }, { + "name" : "version", + "type" : [ "null", "string" ], + "doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.", + "default" : null } ] } ], - "doc" : "A string->string map of custom properties that one might want to attach to an event", + "doc" : "System properties that one might want to attach to an event", + "default" : null + }, { + "name" : "headers", + "type" : [ "null", { + "type" : "map", + "values" : "string" + } ], + "doc" : "Headers - intended to mimic http headers", "default" : null } ] } \ No newline at end of file diff --git a/metadata-integration/java/datahub-protobuf/README.md b/metadata-integration/java/datahub-protobuf/README.md index 29b82aa3e68f5..1b919adb0e3fb 100644 --- a/metadata-integration/java/datahub-protobuf/README.md +++ b/metadata-integration/java/datahub-protobuf/README.md @@ -597,6 +597,8 @@ usage: Proto2DataHub --platform [Optional] The data platform to produce schemas for. e.g. kafka, snowflake, etc. (defaults to kafka) + -protocProp [Optional] Store the protoc as a custom + property. (defaults to false) --slack_id [Optional] The Slack team id if your protobuf files contain comments with references to channel names. We will translate comments like diff --git a/metadata-integration/java/datahub-protobuf/build.gradle b/metadata-integration/java/datahub-protobuf/build.gradle index c8082b875d321..8bc71c5da8f4a 100644 --- a/metadata-integration/java/datahub-protobuf/build.gradle +++ b/metadata-integration/java/datahub-protobuf/build.gradle @@ -29,7 +29,7 @@ dependencies { implementation externalDependency.jgrapht implementation externalDependency.gson implementation externalDependency.commonsCli - implementation externalDependency.httpAsyncClient + implementation externalDependency.httpClient implementation externalDependency.slf4jApi implementation externalDependency.jacksonCore compileOnly externalDependency.lombok @@ -88,7 +88,6 @@ shadowJar { it.name } dependencies { - exclude(dependency { exclude_modules.contains(it.name) }) @@ -115,6 +114,7 @@ shadowJar { relocate 'org.jgrapht', 'datahub.shaded.org.jgrapht' relocate 'org.jheaps', 'datahub.shaded.org.jheaps' relocate 'ch.randelshofer', 'datahub.shaded.ch.randelshofer' + relocate 'org.apache.hc', 'datahub.shaded.org.apache.hc' finalizedBy checkShadowJar } diff --git a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/Proto2DataHub.java b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/Proto2DataHub.java index 429c6d6bfeba4..2b0ad26e86723 100644 --- a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/Proto2DataHub.java +++ b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/Proto2DataHub.java @@ -156,6 +156,13 @@ public class Proto2DataHub { + "(Default is schema)") .build(); + private static final Option OPTION_PROTOC_CUSTOM_PROPERTY = + Option.builder() + .option("protocProp") + .hasArg(false) + .desc("[Optional] Store the protoc as a custom property. (defaults to false)") + .build(); + enum TransportOptions { REST, KAFKA, @@ -172,6 +179,7 @@ static class AppConfig { private final String slackId; private final String dataPlatform; private final String protoc; + private final boolean enableProtocCustomProperty; private final String inputFile; private final String messageName; private final String inputDir; @@ -207,6 +215,7 @@ static class AppConfig { subType = cli.getOptionValue(OPTION_SUBTYPE, "schema").toLowerCase(Locale.ROOT); inputDir = cli.getOptionValue(OPTION_DIR, null); excludePatterns = cli.getOptionValues(OPTION_EXCLUDE_PATTERN); + enableProtocCustomProperty = cli.hasOption(OPTION_PROTOC_CUSTOM_PROPERTY); } private AppConfig validate() throws Exception { @@ -269,7 +278,8 @@ public static void main(String[] args) throws Exception { .addOption(OPTION_TRANSPORT) .addOption(OPTION_FILENAME) .addOption(OPTION_SUBTYPE) - .addOption(OPTION_HELP); + .addOption(OPTION_HELP) + .addOption(OPTION_PROTOC_CUSTOM_PROPERTY); Options firstPassOptions = new Options().addOption(OPTION_HELP); @@ -357,6 +367,7 @@ public static void main(String[] args) throws Exception { ProtobufDataset.builder() .setDataPlatformUrn(new DataPlatformUrn(config.dataPlatform)) .setProtocIn(new FileInputStream(config.protoc)) + .setEnableProtocCustomProperty(config.enableProtocCustomProperty) .setFilename(filePath.toString()) .setSchema(textSchema) .setAuditStamp(auditStamp) diff --git a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/ProtobufDataset.java b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/ProtobufDataset.java index e0c27ebea18bc..7ab9da71e87eb 100644 --- a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/ProtobufDataset.java +++ b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/ProtobufDataset.java @@ -59,6 +59,7 @@ public static class Builder { private String githubOrganization; private String slackTeamId; private String subType; + private boolean enableProtocCustomProperty; public Builder setGithubOrganization(@Nullable String githubOrganization) { this.githubOrganization = githubOrganization; @@ -119,6 +120,11 @@ public Builder setSubType(@Nullable String subType) { return this; } + public Builder setEnableProtocCustomProperty(boolean enableProtocCustomProperty) { + this.enableProtocCustomProperty = enableProtocCustomProperty; + return this; + } + public ProtobufDataset build() throws IOException { FileDescriptorSet fileSet = FileDescriptorSet.parseFrom(protocBytes); @@ -135,6 +141,7 @@ public ProtobufDataset build() throws IOException { .setDatasetVisitor( DatasetVisitor.builder() .protocBase64(Base64.getEncoder().encodeToString(protocBytes)) + .enableProtocCustomProperty(enableProtocCustomProperty) .datasetPropertyVisitors( List.of(new KafkaTopicPropertyVisitor(), new PropertyVisitor())) .institutionalMemoryMetadataVisitors( diff --git a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/visitors/dataset/DatasetVisitor.java b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/visitors/dataset/DatasetVisitor.java index 1b03e13705910..8da4f8280cb57 100644 --- a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/visitors/dataset/DatasetVisitor.java +++ b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/visitors/dataset/DatasetVisitor.java @@ -66,6 +66,8 @@ public class DatasetVisitor @Builder.Default private final ProtobufModelVisitor deprecationVisitor = new DeprecationVisitor(); + @Builder.Default private final boolean enableProtocCustomProperty = false; + @Override public Stream> visitGraph( VisitContext context) { @@ -92,7 +94,9 @@ public Stream> visitGrap .setCustomProperties( new StringMap( Stream.concat( - Stream.of(Map.entry("protoc", protocBase64)), + enableProtocCustomProperty + ? Stream.of(Map.entry("protoc", protocBase64)) + : Stream.empty(), g.accept(context, datasetPropertyVisitors) .flatMap( props -> diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DatasetVisitorTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DatasetVisitorTest.java index 6e99599c852b4..ebf44730576bc 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DatasetVisitorTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DatasetVisitorTest.java @@ -2,6 +2,7 @@ import static datahub.protobuf.TestFixtures.*; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; import com.linkedin.common.urn.DatasetUrn; import com.linkedin.data.template.RecordTemplate; @@ -21,7 +22,8 @@ public class DatasetVisitorTest { @Test public void protocBase64Test() throws URISyntaxException, IOException { String expected = "23454345452345233455"; - DatasetVisitor test = DatasetVisitor.builder().protocBase64(expected).build(); + DatasetVisitor test = + DatasetVisitor.builder().protocBase64(expected).enableProtocCustomProperty(true).build(); List> changes = test.visitGraph( @@ -35,6 +37,23 @@ public void protocBase64Test() throws URISyntaxException, IOException { .collect(Collectors.toList()); assertEquals(expected, extractCustomProperty(changes.get(0), "protoc")); + + DatasetVisitor testDisabled = + DatasetVisitor.builder().protocBase64(expected).enableProtocCustomProperty(false).build(); + + List> changesDisabled = + testDisabled + .visitGraph( + VisitContext.builder() + .auditStamp(TEST_AUDIT_STAMP) + .datasetUrn( + DatasetUrn.createFromString( + "urn:li:dataset:(urn:li:dataPlatform:kafka,protobuf.MessageA,TEST)")) + .graph(getTestProtobufGraph("protobuf", "messageA")) + .build()) + .collect(Collectors.toList()); + + assertNull(extractCustomProperty(changesDisabled.get(0), "protoc")); } @Test diff --git a/metadata-integration/java/examples/build.gradle b/metadata-integration/java/examples/build.gradle index 62c80562c7c3b..43e232b4e4bfa 100644 --- a/metadata-integration/java/examples/build.gradle +++ b/metadata-integration/java/examples/build.gradle @@ -12,7 +12,7 @@ dependencies { implementation externalDependency.opentracingJdbc implementation project(path: ':metadata-integration:java:datahub-client', configuration: 'shadow') - implementation externalDependency.httpAsyncClient + implementation externalDependency.httpClient // Tests need a concrete log4j available. Providing it here testImplementation 'org.apache.logging.log4j:log4j-api:2.17.1' diff --git a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetStructuredPropertiesUpdate.java b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetStructuredPropertiesUpdate.java new file mode 100644 index 0000000000000..c6c15d7a068ee --- /dev/null +++ b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetStructuredPropertiesUpdate.java @@ -0,0 +1,79 @@ +package io.datahubproject.examples; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.aspect.patch.builder.StructuredPropertiesPatchBuilder; +import com.linkedin.mxe.MetadataChangeProposal; +import datahub.client.MetadataWriteResponse; +import datahub.client.rest.RestEmitter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class DatasetStructuredPropertiesUpdate { + + private DatasetStructuredPropertiesUpdate() {} + + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException { + + // Adding a structured property with a single string value + MetadataChangeProposal mcp1 = + new StructuredPropertiesPatchBuilder() + .urn( + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)")) + .setStringProperty( + UrnUtils.getUrn("urn:li:structuredProperty:io.acryl.privacy.retentionTime"), "30") + .build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + Future response1 = emitter.emit(mcp1, null); + System.out.println(response1.get().getResponseContent()); + + // Adding a structured property with a list of string values + List values = new ArrayList<>(); + values.add("30"); + values.add("90"); + MetadataChangeProposal mcp2 = + new StructuredPropertiesPatchBuilder() + .urn( + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)")) + .setStringProperty( + UrnUtils.getUrn("urn:li:structuredProperty:io.acryl.privacy.retentionTime"), values) + .build(); + + Future response2 = emitter.emit(mcp2, null); + System.out.println(response2.get().getResponseContent()); + + // Adding a structured property with a single number value + MetadataChangeProposal mcp3 = + new StructuredPropertiesPatchBuilder() + .urn( + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)")) + .setNumberProperty( + UrnUtils.getUrn("urn:li:structuredProperty:io.acryl.dataManagement.replicationSLA"), + 3456) + .build(); + + Future response3 = emitter.emit(mcp3, null); + System.out.println(response3.get().getResponseContent()); + + // Removing a structured property from a dataset + MetadataChangeProposal mcp4 = + new StructuredPropertiesPatchBuilder() + .urn( + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)")) + .removeProperty( + UrnUtils.getUrn("urn:li:structuredProperty:io.acryl.dataManagement.replicationSLA")) + .build(); + + Future response4 = emitter.emit(mcp4, null); + System.out.println(response4.get().getResponseContent()); + } +} diff --git a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/FormCreate.java b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/FormCreate.java new file mode 100644 index 0000000000000..5451d431b3b99 --- /dev/null +++ b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/FormCreate.java @@ -0,0 +1,68 @@ +package io.datahubproject.examples; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.form.FormActorAssignment; +import com.linkedin.form.FormInfo; +import com.linkedin.form.FormPrompt; +import com.linkedin.form.FormPromptArray; +import com.linkedin.form.FormPromptType; +import com.linkedin.form.FormType; +import com.linkedin.form.StructuredPropertyParams; +import datahub.client.MetadataWriteResponse; +import datahub.client.rest.RestEmitter; +import datahub.event.MetadataChangeProposalWrapper; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class FormCreate { + + private FormCreate() {} + + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException { + FormPromptArray prompts = new FormPromptArray(); + FormPrompt prompt1 = + new FormPrompt() + .setId("1") + .setTitle("First Prompt") + .setType(FormPromptType.STRUCTURED_PROPERTY) + .setRequired(true) + .setStructuredPropertyParams( + new StructuredPropertyParams() + .setUrn(UrnUtils.getUrn("urn:li:structuredProperty:property1"))); + FormPrompt prompt2 = + new FormPrompt() + .setId("2") + .setTitle("Second Prompt") + .setType(FormPromptType.FIELDS_STRUCTURED_PROPERTY) + .setRequired(false) + .setStructuredPropertyParams( + new StructuredPropertyParams() + .setUrn(UrnUtils.getUrn("urn:li:structuredProperty:property1"))); + prompts.add(prompt1); + prompts.add(prompt2); + + FormInfo formInfo = + new FormInfo() + .setName("Metadata Initiative 2024") + .setDescription("Please respond to this form for metadata compliance purposes.") + .setType(FormType.VERIFICATION) + .setPrompts(prompts) + .setActors(new FormActorAssignment().setOwners(true)); + + MetadataChangeProposalWrapper mcpw = + MetadataChangeProposalWrapper.builder() + .entityType("form") + .entityUrn("urn:li:form:metadata_initiative_1") + .upsert() + .aspect(formInfo) + .aspectName("formInfo") + .build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + Future response = emitter.emit(mcpw, null); + System.out.println(response.get().getResponseContent()); + } +} diff --git a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/FormUpdate.java b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/FormUpdate.java new file mode 100644 index 0000000000000..d986f8d403ef6 --- /dev/null +++ b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/FormUpdate.java @@ -0,0 +1,57 @@ +package io.datahubproject.examples; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.form.FormPrompt; +import com.linkedin.form.FormPromptType; +import com.linkedin.form.StructuredPropertyParams; +import com.linkedin.metadata.aspect.patch.builder.FormInfoPatchBuilder; +import com.linkedin.mxe.MetadataChangeProposal; +import datahub.client.MetadataWriteResponse; +import datahub.client.rest.RestEmitter; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class FormUpdate { + + private FormUpdate() {} + + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException { + FormPrompt newPrompt = + new FormPrompt() + .setId("1234") + .setTitle("First Prompt") + .setType(FormPromptType.STRUCTURED_PROPERTY) + .setRequired(true) + .setStructuredPropertyParams( + new StructuredPropertyParams() + .setUrn(UrnUtils.getUrn("urn:li:structuredProperty:property1"))); + FormPrompt newPrompt2 = + new FormPrompt() + .setId("abcd") + .setTitle("Second Prompt") + .setType(FormPromptType.FIELDS_STRUCTURED_PROPERTY) + .setRequired(false) + .setStructuredPropertyParams( + new StructuredPropertyParams() + .setUrn(UrnUtils.getUrn("urn:li:structuredProperty:property1"))); + + Urn formUrn = UrnUtils.getUrn("urn:li:form:metadata_initiative_1"); + FormInfoPatchBuilder formInfoPatchBuilder = + new FormInfoPatchBuilder() + .urn(formUrn) + .addPrompts(List.of(newPrompt, newPrompt2)) + .setName("Metadata Initiative 2024 (edited)") + .setDescription("Edited description") + .setOwnershipForm(true); + MetadataChangeProposal mcp = formInfoPatchBuilder.build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + Future response = emitter.emit(mcp, null); + System.out.println(response.get().getResponseContent()); + } +} diff --git a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/StructuredPropertyUpsert.java b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/StructuredPropertyUpsert.java new file mode 100644 index 0000000000000..6d6bd232bf0bc --- /dev/null +++ b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/StructuredPropertyUpsert.java @@ -0,0 +1,102 @@ +package io.datahubproject.examples; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringArray; +import com.linkedin.data.template.StringArrayMap; +import com.linkedin.metadata.aspect.patch.builder.StructuredPropertyDefinitionPatchBuilder; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.structured.PrimitivePropertyValue; +import com.linkedin.structured.PropertyCardinality; +import com.linkedin.structured.PropertyValue; +import datahub.client.MetadataWriteResponse; +import datahub.client.rest.RestEmitter; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class StructuredPropertyUpsert { + + private StructuredPropertyUpsert() {} + + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException { + // open ended string structured property on datasets and dataFlows + MetadataChangeProposal mcp1 = + new StructuredPropertyDefinitionPatchBuilder() + .urn( + UrnUtils.getUrn( + "urn:li:structuredProperty:testString")) // use existing urn for update, new urn + // for new property + .setQualifiedName("io.acryl.testString") + .setDisplayName("Open Ended String") + .setValueType("urn:li:dataType:datahub.string") + .setCardinality(PropertyCardinality.SINGLE) + .addEntityType("urn:li:entityType:datahub.dataset") + .addEntityType("urn:li:entityType:datahub.dataFlow") + .setDescription("test description for open ended string") + .setImmutable(true) + .build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + Future response1 = emitter.emit(mcp1, null); + System.out.println(response1.get().getResponseContent()); + + // Next, let's make a property that allows for multiple datahub entity urns as values + // This example property could be used to reference other users or groups in datahub + StringArrayMap typeQualifier = new StringArrayMap(); + typeQualifier.put( + "allowedTypes", + new StringArray( + "urn:li:entityType:datahub.corpuser", "urn:li:entityType:datahub.corpGroup")); + + MetadataChangeProposal mcp2 = + new StructuredPropertyDefinitionPatchBuilder() + .urn(UrnUtils.getUrn("urn:li:structuredProperty:dataSteward")) + .setQualifiedName("io.acryl.dataManagement.dataSteward") + .setDisplayName("Data Steward") + .setValueType("urn:li:dataType:datahub.urn") + .setTypeQualifier(typeQualifier) + .setCardinality(PropertyCardinality.MULTIPLE) + .addEntityType("urn:li:entityType:datahub.dataset") + .setDescription( + "The data stewards of this asset are in charge of ensuring data cleanliness and governance") + .setImmutable(true) + .build(); + + Future response2 = emitter.emit(mcp2, null); + System.out.println(response2.get().getResponseContent()); + + // Finally, let's make a single select number property with a few allowed options + PropertyValue propertyValue1 = new PropertyValue(); + PrimitivePropertyValue value1 = new PrimitivePropertyValue(); + value1.setDouble(30.0); + propertyValue1.setDescription( + "30 days, usually reserved for datasets that are ephemeral and contain pii"); + propertyValue1.setValue(value1); + PropertyValue propertyValue2 = new PropertyValue(); + PrimitivePropertyValue value2 = new PrimitivePropertyValue(); + value2.setDouble(90.0); + propertyValue2.setDescription( + "Use this for datasets that drive monthly reporting but contain pii"); + propertyValue2.setValue(value2); + + MetadataChangeProposal mcp3 = + new StructuredPropertyDefinitionPatchBuilder() + .urn(UrnUtils.getUrn("urn:li:structuredProperty:replicationSLA")) + .setQualifiedName("io.acryl.dataManagement.replicationSLA") + .setDisplayName("Replication SLA") + .setValueType("urn:li:dataType:datahub.string") + .addAllowedValue(propertyValue1) + .addAllowedValue(propertyValue2) + .setCardinality(PropertyCardinality.SINGLE) + .addEntityType("urn:li:entityType:datahub.dataset") + .setDescription( + "SLA for how long data can be delayed before replicating to the destination cluster") + .setImmutable(false) + .build(); + + Future response3 = emitter.emit(mcp3, null); + System.out.println(response3.get().getResponseContent()); + } +} diff --git a/metadata-integration/java/openlineage-converter/build.gradle b/metadata-integration/java/openlineage-converter/build.gradle index 4fb35f6b3c563..145a88229ad4a 100644 --- a/metadata-integration/java/openlineage-converter/build.gradle +++ b/metadata-integration/java/openlineage-converter/build.gradle @@ -24,9 +24,7 @@ dependencies { implementation externalDependency.json - testImplementation externalDependency.junit - // Use JUnit Jupiter for testing. - testImplementation 'org.junit.jupiter:junit-jupiter:5.9.2' + testImplementation externalDependency.testng } jacocoTestReport { diff --git a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/config/DatahubOpenlineageConfig.java b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/config/DatahubOpenlineageConfig.java index 5e4b791fa8d82..5abb3c90d232b 100644 --- a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/config/DatahubOpenlineageConfig.java +++ b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/config/DatahubOpenlineageConfig.java @@ -10,9 +10,11 @@ import java.util.stream.Collectors; import lombok.Builder; import lombok.Getter; +import lombok.ToString; @Builder @Getter +@ToString public class DatahubOpenlineageConfig { @Builder.Default private final boolean isStreaming = false; @Builder.Default private final String pipelineName = null; @@ -28,6 +30,10 @@ public class DatahubOpenlineageConfig { @Builder.Default private final DataJobUrn parentJobUrn = null; // This is disabled until column level patch support won't be fixed in GMS @Builder.Default private final boolean usePatch = true; + @Builder.Default private String hivePlatformAlias = "hive"; + @Builder.Default private Map urnAliases = new HashMap<>(); + @Builder.Default private final boolean disableSymlinkResolution = false; + @Builder.Default private final boolean lowerCaseDatasetUrns = false; public List getPathSpecsForPlatform(String platform) { if ((pathSpecs == null) || (pathSpecs.isEmpty())) { diff --git a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/converter/OpenLineageToDataHub.java b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/converter/OpenLineageToDataHub.java index 1db09306cbdc2..1568451beff10 100644 --- a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/converter/OpenLineageToDataHub.java +++ b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/converter/OpenLineageToDataHub.java @@ -4,6 +4,7 @@ import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Edge; import com.linkedin.common.EdgeArray; +import com.linkedin.common.FabricType; import com.linkedin.common.GlobalTags; import com.linkedin.common.Owner; import com.linkedin.common.OwnerArray; @@ -57,6 +58,8 @@ import io.datahubproject.openlineage.dataset.DatahubJob; import io.datahubproject.openlineage.dataset.HdfsPathDataset; import io.datahubproject.openlineage.dataset.HdfsPlatform; +import io.datahubproject.openlineage.dataset.PathSpec; +import io.datahubproject.openlineage.utils.DatahubUtils; import io.openlineage.client.OpenLineage; import io.openlineage.client.OpenLineageClientUtils; import java.io.IOException; @@ -98,10 +101,64 @@ private OpenLineageToDataHub() {} public static Optional convertOpenlineageDatasetToDatasetUrn( OpenLineage.Dataset dataset, DatahubOpenlineageConfig mappingConfig) { + String namespace = dataset.getNamespace(); String datasetName = dataset.getName(); + Optional datahubUrn; + if (dataset.getFacets() != null + && dataset.getFacets().getSymlinks() != null + && !mappingConfig.isDisableSymlinkResolution()) { + Optional originalUrn = + getDatasetUrnFromOlDataset(namespace, datasetName, mappingConfig); + for (OpenLineage.SymlinksDatasetFacetIdentifiers symlink : + dataset.getFacets().getSymlinks().getIdentifiers()) { + if (symlink.getType().equals("TABLE")) { + if (symlink.getNamespace().startsWith("aws:glue:")) { + namespace = "glue"; + } else { + namespace = mappingConfig.getHivePlatformAlias(); + } + datasetName = symlink.getName(); + } + } + Optional symlinkedUrn = + getDatasetUrnFromOlDataset(namespace, datasetName, mappingConfig); + if (symlinkedUrn.isPresent() && originalUrn.isPresent()) { + mappingConfig + .getUrnAliases() + .put(originalUrn.get().toString(), symlinkedUrn.get().toString()); + } + datahubUrn = symlinkedUrn; + } else { + datahubUrn = getDatasetUrnFromOlDataset(namespace, datasetName, mappingConfig); + } + log.debug("Dataset URN: {}, alias_list: {}", datahubUrn, mappingConfig.getUrnAliases()); + // If we have the urn in urn aliases then we should use the alias instead of the original urn + if (datahubUrn.isPresent() + && mappingConfig.getUrnAliases().containsKey(datahubUrn.get().toString())) { + try { + datahubUrn = + Optional.of( + DatasetUrn.createFromString( + mappingConfig.getUrnAliases().get(datahubUrn.get().toString()))); + return datahubUrn; + } catch (URISyntaxException e) { + return Optional.empty(); + } + } + + return datahubUrn; + } + + private static Optional getDatasetUrnFromOlDataset( + String namespace, String datasetName, DatahubOpenlineageConfig mappingConfig) { String platform; + if (mappingConfig.isLowerCaseDatasetUrns()) { + namespace = namespace.toLowerCase(); + datasetName = datasetName.toLowerCase(); + } + if (namespace.contains(SCHEME_SEPARATOR)) { try { URI datasetUri; @@ -115,8 +172,8 @@ public static Optional convertOpenlineageDatasetToDatasetUrn( } else { platform = datasetUri.getScheme(); } - datasetName = datasetUri.getPath(); if (HdfsPlatform.isFsPlatformPrefix(platform)) { + datasetName = datasetUri.getPath(); try { HdfsPathDataset hdfsPathDataset = HdfsPathDataset.create(datasetUri, mappingConfig); return Optional.of(hdfsPathDataset.urn()); @@ -125,8 +182,6 @@ public static Optional convertOpenlineageDatasetToDatasetUrn( "Unable to create urn from namespace: {} and dataset {}.", namespace, datasetName); return Optional.empty(); } - } else { - datasetName = dataset.getName(); } } catch (URISyntaxException e) { log.warn("Unable to create URI from namespace: {} and dataset {}.", namespace, datasetName); @@ -134,15 +189,47 @@ public static Optional convertOpenlineageDatasetToDatasetUrn( } } else { platform = namespace; - datasetName = dataset.getName(); } - if (mappingConfig.getCommonDatasetPlatformInstance() != null) { - datasetName = mappingConfig.getCommonDatasetPlatformInstance() + "." + datasetName; + String platformInstance = getPlatformInstance(mappingConfig, platform); + FabricType env = getEnv(mappingConfig, platform); + return Optional.of(DatahubUtils.createDatasetUrn(platform, platformInstance, datasetName, env)); + } + + private static FabricType getEnv(DatahubOpenlineageConfig mappingConfig, String platform) { + FabricType fabricType = mappingConfig.getFabricType(); + if (mappingConfig.getPathSpecs() != null + && mappingConfig.getPathSpecs().containsKey(platform)) { + List path_specs = mappingConfig.getPathSpecs().get(platform); + for (PathSpec pathSpec : path_specs) { + if (pathSpec.getEnv().isPresent()) { + try { + fabricType = FabricType.valueOf(pathSpec.getEnv().get()); + return fabricType; + } catch (IllegalArgumentException e) { + log.warn("Invalid environment value: {}", pathSpec.getEnv()); + } + } + } } + return fabricType; + } - return Optional.of( - new DatasetUrn(new DataPlatformUrn(platform), datasetName, mappingConfig.getFabricType())); + private static String getPlatformInstance( + DatahubOpenlineageConfig mappingConfig, String platform) { + // Use the platform instance from the path spec if it is present otherwise use the one from the + // commonDatasetPlatformInstance + String platformInstance = mappingConfig.getCommonDatasetPlatformInstance(); + if (mappingConfig.getPathSpecs() != null + && mappingConfig.getPathSpecs().containsKey(platform)) { + List path_specs = mappingConfig.getPathSpecs().get(platform); + for (PathSpec pathSpec : path_specs) { + if (pathSpec.getPlatformInstance().isPresent()) { + return pathSpec.getPlatformInstance().get(); + } + } + } + return platformInstance; } public static GlobalTags generateTags(List tags) { @@ -328,8 +415,15 @@ private static UpstreamLineage getFineGrainedLineage( + inputField.getField() + ")"); upstreamFields.add(datasetFieldUrn); - upstreams.add( - new Upstream().setDataset(urn.get()).setType(DatasetLineageType.TRANSFORMED)); + if (upstreams.stream() + .noneMatch( + upstream -> + upstream.getDataset().toString().equals(urn.get().toString()))) { + upstreams.add( + new Upstream() + .setDataset(urn.get()) + .setType(DatasetLineageType.TRANSFORMED)); + } } }); @@ -530,6 +624,7 @@ private static void convertJobToDataJob( OpenLineage.Job job = event.getJob(); DataJobInfo dji = new DataJobInfo(); + log.debug("Datahub Config: {}", datahubConf); if (job.getName().contains(".")) { String jobName = job.getName().substring(job.getName().indexOf(".") + 1); @@ -675,7 +770,7 @@ private static void processJobInputs( DatahubDataset.DatahubDatasetBuilder builder = DatahubDataset.builder(); builder.urn(datasetUrn.get()); if (datahubConf.isMaterializeDataset()) { - builder.schemaMetadata(getSchemaMetadata(input)); + builder.schemaMetadata(getSchemaMetadata(input, datahubConf)); } if (datahubConf.isCaptureColumnLevelLineage()) { UpstreamLineage upstreamLineage = getFineGrainedLineage(input, datahubConf); @@ -705,7 +800,7 @@ private static void processJobOutputs( DatahubDataset.DatahubDatasetBuilder builder = DatahubDataset.builder(); builder.urn(datasetUrn.get()); if (datahubConf.isMaterializeDataset()) { - builder.schemaMetadata(getSchemaMetadata(output)); + builder.schemaMetadata(getSchemaMetadata(output, datahubConf)); } if (datahubConf.isCaptureColumnLevelLineage()) { UpstreamLineage upstreamLineage = getFineGrainedLineage(output, datahubConf); @@ -836,7 +931,8 @@ public static SchemaFieldDataType.Type convertOlFieldTypeToDHFieldType( } } - public static SchemaMetadata getSchemaMetadata(OpenLineage.Dataset dataset) { + public static SchemaMetadata getSchemaMetadata( + OpenLineage.Dataset dataset, DatahubOpenlineageConfig mappingConfig) { SchemaFieldArray schemaFieldArray = new SchemaFieldArray(); if ((dataset.getFacets() == null) || (dataset.getFacets().getSchema() == null)) { return null; @@ -865,9 +961,16 @@ public static SchemaMetadata getSchemaMetadata(OpenLineage.Dataset dataset) { ddl.setTableSchema(OpenLineageClientUtils.toJson(dataset.getFacets().getSchema().getFields())); SchemaMetadata.PlatformSchema platformSchema = new SchemaMetadata.PlatformSchema(); platformSchema.setMySqlDDL(ddl); + Optional datasetUrn = + getDatasetUrnFromOlDataset(dataset.getNamespace(), dataset.getName(), mappingConfig); + + if (!datasetUrn.isPresent()) { + return null; + } + schemaMetadata.setPlatformSchema(platformSchema); - schemaMetadata.setPlatform(new DataPlatformUrn(dataset.getNamespace())); + schemaMetadata.setPlatform(datasetUrn.get().getPlatformEntity()); schemaMetadata.setFields(schemaFieldArray); return schemaMetadata; diff --git a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/DatahubJob.java b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/DatahubJob.java index 5f4a9b6a596e7..60caaae359677 100644 --- a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/DatahubJob.java +++ b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/DatahubJob.java @@ -189,7 +189,9 @@ private void generateDataJobInputOutputMcp( MetadataChangeProposal dataJobInputOutputMcp = dataJobInputOutputPatchBuilder.build(); log.info( "dataJobInputOutputMcp: {}", - dataJobInputOutputMcp.getAspect().getValue().asString(Charset.defaultCharset())); + Objects.requireNonNull(dataJobInputOutputMcp.getAspect()) + .getValue() + .asString(Charset.defaultCharset())); mcps.add(dataJobInputOutputPatchBuilder.build()); } else { @@ -278,11 +280,11 @@ private Pair processDownstreams( for (Urn downstream : Objects.requireNonNull(fineGrainedLineage.getDownstreams())) { upstreamLineagePatchBuilder.addFineGrainedUpstreamField( - downstream, + upstream, fineGrainedLineage.getConfidenceScore(), StringUtils.defaultIfEmpty( fineGrainedLineage.getTransformOperation(), "TRANSFORM"), - upstream, + downstream, null); } } @@ -292,9 +294,9 @@ private Pair processDownstreams( "upstreamLineagePatch: {}", mcp.getAspect().getValue().asString(Charset.defaultCharset())); mcps.add(mcp); - } else { - addAspectToMcps(dataset.getUrn(), DATASET_ENTITY_TYPE, dataset.getLineage(), mcps); } + } else { + addAspectToMcps(dataset.getUrn(), DATASET_ENTITY_TYPE, dataset.getLineage(), mcps); } } }); diff --git a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/HdfsPathDataset.java b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/HdfsPathDataset.java index 0d0868afedfd9..b938db24fc626 100644 --- a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/HdfsPathDataset.java +++ b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/HdfsPathDataset.java @@ -69,8 +69,7 @@ public static HdfsPathDataset create(URI path, DatahubOpenlineageConfig datahubC String rawName = getRawNameFromUri(pathUri, pathSpec.getPathSpecList()); if (rawName != null) { - String platformInstance = - pathSpec.platformInstance.orElseGet(datahubConf::getCommonDatasetPlatformInstance); + String platformInstance = pathSpec.platformInstance.orElse(null); FabricType fabricType = datahubConf.getFabricType(); return new HdfsPathDataset( platform, getDatasetName(rawName), platformInstance, fabricType, rawName); diff --git a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/PathSpec.java b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/PathSpec.java index 1a015cabc46cc..b73b176321603 100644 --- a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/PathSpec.java +++ b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/PathSpec.java @@ -14,7 +14,7 @@ public class PathSpec { final String alias; final String platform; - @Builder.Default final String env = "PROD"; + @Builder.Default final Optional env = Optional.empty(); final List pathSpecList; @Builder.Default final Optional platformInstance = Optional.empty(); } diff --git a/metadata-integration/java/openlineage-converter/src/test/java/io/datahubproject/openlineage/HdfsPathDatasetTest.java b/metadata-integration/java/openlineage-converter/src/test/java/io/datahubproject/openlineage/HdfsPathDatasetTest.java index e8981aeb0be59..82ba172f84191 100644 --- a/metadata-integration/java/openlineage-converter/src/test/java/io/datahubproject/openlineage/HdfsPathDatasetTest.java +++ b/metadata-integration/java/openlineage-converter/src/test/java/io/datahubproject/openlineage/HdfsPathDatasetTest.java @@ -13,8 +13,8 @@ import java.util.LinkedList; import java.util.List; import java.util.Optional; -import org.junit.Assert; -import org.junit.Test; +import org.testng.Assert; +import org.testng.annotations.Test; public class HdfsPathDatasetTest { @@ -42,7 +42,7 @@ public void testPathSpecList() "s3", Collections.singletonList( PathSpec.builder() - .env("PROD") + .env(Optional.of("PROD")) .platform("s3") .pathSpecList( new LinkedList<>( @@ -89,7 +89,7 @@ public void testNoMatchPathSpecList() "s3", Collections.singletonList( PathSpec.builder() - .env("PROD") + .env(Optional.of("PROD")) .platform("s3") .pathSpecList( new LinkedList<>( @@ -121,7 +121,7 @@ public void testPathSpecListPlatformInstance() "s3", Collections.singletonList( PathSpec.builder() - .env("PROD") + .env(Optional.of("PROD")) .platform("s3") .pathSpecList( new LinkedList<>( @@ -137,7 +137,7 @@ public void testPathSpecListPlatformInstance() SparkDataset dataset = HdfsPathDataset.create(new URI("s3a://my-bucket/foo/tests/bar.avro"), datahubConfig); Assert.assertEquals( - "urn:li:dataset:(urn:li:dataPlatform:s3,instance.my-bucket/foo/tests,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:s3,my-bucket/foo/tests,PROD)", dataset.urn().toString()); } @@ -154,7 +154,7 @@ public void testPathSpecListPathSpecPlatformInstance() "s3", Collections.singletonList( PathSpec.builder() - .env("PROD") + .env(Optional.of("PROD")) .platform("s3") .platformInstance(Optional.of("s3Instance")) .pathSpecList( @@ -187,7 +187,7 @@ public void testPathAliasList() "s3", Collections.singletonList( PathSpec.builder() - .env("PROD") + .env(Optional.of("PROD")) .platform("s3") .pathSpecList( new LinkedList<>( @@ -219,7 +219,7 @@ public void testGcsNoPathSpecList() "s3", Collections.singletonList( PathSpec.builder() - .env("PROD") + .env(Optional.of("PROD")) .platform("gcs") .pathSpecList( new LinkedList<>( @@ -249,7 +249,7 @@ public void testGcsPathSpecList() "s3", Collections.singletonList( PathSpec.builder() - .env("PROD") + .env(Optional.of("PROD")) .platform("gcs") .pathSpecList( new LinkedList<>( diff --git a/metadata-integration/java/spark-lineage-beta/README.md b/metadata-integration/java/spark-lineage-beta/README.md index 7b3598453498f..a643919664b07 100644 --- a/metadata-integration/java/spark-lineage-beta/README.md +++ b/metadata-integration/java/spark-lineage-beta/README.md @@ -24,7 +24,7 @@ When running jobs using spark-submit, the agent needs to be configured in the co ```text #Configuring DataHub spark agent jar -spark.jars.packages io.acryl:acryl-spark-lineage:0.2.6 +spark.jars.packages io.acryl:acryl-spark-lineage:0.2.13 spark.extraListeners datahub.spark.DatahubSparkListener spark.datahub.rest.server http://localhost:8080 ``` @@ -32,7 +32,7 @@ spark.datahub.rest.server http://localhost:8080 ## spark-submit command line ```sh -spark-submit --packages io.acryl:acryl-spark-lineage:0.2.6 --conf "spark.extraListeners=datahub.spark.DatahubSparkListener" my_spark_job_to_run.py +spark-submit --packages io.acryl:acryl-spark-lineage:0.2.13 --conf "spark.extraListeners=datahub.spark.DatahubSparkListener" my_spark_job_to_run.py ``` ### Configuration Instructions: Amazon EMR @@ -41,7 +41,7 @@ Set the following spark-defaults configuration properties as it stated [here](https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-spark-configure.html) ```text -spark.jars.packages io.acryl:acryl-spark-lineage:0.2.6 +spark.jars.packages io.acryl:acryl-spark-lineage:0.2.13 spark.extraListeners datahub.spark.DatahubSparkListener spark.datahub.rest.server https://your_datahub_host/gms #If you have authentication set up then you also need to specify the Datahub access token @@ -56,7 +56,7 @@ When running interactive jobs from a notebook, the listener can be configured wh spark = SparkSession.builder .master("spark://spark-master:7077") .appName("test-application") -.config("spark.jars.packages", "io.acryl:acryl-spark-lineage:0.2.6") +.config("spark.jars.packages", "io.acryl:acryl-spark-lineage:0.2.13") .config("spark.extraListeners", "datahub.spark.DatahubSparkListener") .config("spark.datahub.rest.server", "http://localhost:8080") .enableHiveSupport() @@ -79,7 +79,7 @@ appName("test-application") config("spark.master","spark://spark-master:7077") . -config("spark.jars.packages","io.acryl:acryl-spark-lineage:0.2.6") +config("spark.jars.packages","io.acryl:acryl-spark-lineage:0.2.13") . config("spark.extraListeners","datahub.spark.DatahubSparkListener") @@ -159,17 +159,17 @@ information like tokens. | Field | Required | Default | Description | |---------------------------------------------------------------------|----------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| spark.jars.packages | ✅ | | Set with latest/required version io.acryl:datahub-spark-lineage:0.8.23 | +| spark.jars.packages | ✅ | | Set with latest/required version io.acryl:acryl-spark-lineage:0.2.13 | | spark.extraListeners | ✅ | | datahub.spark.DatahubSparkListener | | spark.datahub.rest.server | ✅ | | Datahub server url eg: | | spark.datahub.rest.token | | | Authentication token. | | spark.datahub.rest.disable_ssl_verification | | false | Disable SSL certificate validation. Caution: Only use this if you know what you are doing! | -| spark.datahub.rest.rest.max_retries | | 0 | Number of times a request retried if failed | -| spark.datahub.rest.rest.retry_interval | | 10 | Number of seconds to wait between retries | +| spark.datahub.rest.max_retries | | 0 | Number of times a request retried if failed | +| spark.datahub.rest.retry_interval | | 10 | Number of seconds to wait between retries | | spark.datahub.metadata.pipeline.platformInstance | | | Pipeline level platform instance | -| spark.datahub.metadata.dataset.platformInstance | | | dataset level platform instance | +| spark.datahub.metadata.dataset.platformInstance | | | dataset level platform instance (it is usefult to set if you have it in your glue ingestion) | | spark.datahub.metadata.dataset.env | | PROD | [Supported values](https://datahubproject.io/docs/graphql/enums#fabrictype). In all other cases, will fallback to PROD | -| spark.datahub.metadata.table.hive_platform_alias | | hive | By default, datahub assigns Hive-like tables to the Hive platform. If you are using Glue as your Hive metastore, set this config flag to `glue` | +| spark.datahub.metadata.dataset.hivePlatformAlias | | hive | By default, datahub assigns Hive-like tables to the Hive platform. If you are using Glue as your Hive metastore, set this config flag to `glue` | | spark.datahub.metadata.include_scheme | | true | Include scheme from the path URI (e.g. hdfs://, s3://) in the dataset URN. We recommend setting this value to false, it is set to true for backwards compatibility with previous versions | | spark.datahub.metadata.remove_partition_pattern | | | Remove partition pattern. (e.g. /partition=\d+) It change database/table/partition=123 to database/table | | spark.datahub.coalesce_jobs | | true | Only one datajob(task) will be emitted containing all input and output datasets for the spark application | @@ -181,9 +181,10 @@ information like tokens. | spark.datahub.partition_regexp_pattern | | | Strip partition part from the path if path end matches with the specified regexp. Example `year=.*/month=.*/day=.*` | | spark.datahub.tags | | | Comma separated list of tags to attach to the DataFlow | | spark.datahub.domains | | | Comma separated list of domain urns to attach to the DataFlow | -| spark.datahub.stage_metadata_coalescing | | | Normally it coalesce and send metadata at the onApplicationEnd event which is never called on Databricks. You should enable this on Databricks if you want coalesced run . | -| spark.datahub.patch.enabled | | true | Set this to true to send lineage as a patch, which appends rather than overwrites existing Dataset lineage edges. By default it is enabled. -| +| spark.datahub.stage_metadata_coalescing | | | Normally it coalesces and sends metadata at the onApplicationEnd event which is never called on Databricks or on Glue. You should enable this on Databricks if you want coalesced run. | +| spark.datahub.patch.enabled | | false | Set this to true to send lineage as a patch, which appends rather than overwrites existing Dataset lineage edges. By default, it is disabled. | +| spark.datahub.metadata.dataset.lowerCaseUrns | | false | Set this to true to lowercase dataset urns. By default, it is disabled. | +| spark.datahub.disableSymlinkResolution | | false | Set this to true if you prefer using the s3 location instead of the Hive table. By default, it is disabled. | ## What to Expect: The Metadata Model @@ -343,3 +344,15 @@ Use Java 8 to build the project. The project uses Gradle as the build tool. To b ``` ## Known limitations ++ +## Changelog + +### Version 0.2.12 +- Silencing some chatty warnings in RddPathUtils + +### Version 0.2.12 + +- Add option to lowercase dataset URNs +- Add option to set platform instance and/or env per platform with `spark.datahub.platform..env` and `spark.datahub.platform..platform_instance` config parameter +- Fixing platform instance setting for datasets when `spark.datahub.metadata.dataset.platformInstance` is set +- Fixing column level lineage support when patch is enabled diff --git a/metadata-integration/java/spark-lineage-beta/build.gradle b/metadata-integration/java/spark-lineage-beta/build.gradle index d83753028d0b4..99b87b9b89bf4 100644 --- a/metadata-integration/java/spark-lineage-beta/build.gradle +++ b/metadata-integration/java/spark-lineage-beta/build.gradle @@ -56,6 +56,7 @@ dependencies { implementation "io.openlineage:openlineage-spark_2.12:$openLineageVersion" compileOnly "org.apache.iceberg:iceberg-spark3-runtime:0.12.1" compileOnly "org.apache.spark:spark-sql_2.12:3.1.3" + compileOnly "io.github.spark-redshift-community:spark-redshift_2.12:6.2.0-spark_3.5" testCompileOnly externalDependency.lombok testAnnotationProcessor externalDependency.lombok diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/DatahubEventEmitter.java b/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/DatahubEventEmitter.java index 1dc086e4af585..5a3f4bd27b415 100644 --- a/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/DatahubEventEmitter.java +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/DatahubEventEmitter.java @@ -176,7 +176,7 @@ public List generateCoalescedMcps() { AtomicLong maxEndTime = new AtomicLong(); _datahubJobs.forEach( storedDatahubJob -> { - log.info("Merging job stored job {} to {}", storedDatahubJob, datahubJob); + log.info("Merging job stored job {} with {}", storedDatahubJob, datahubJob); DataJobUrn jobUrn = jobUrn( storedDatahubJob.getFlowUrn(), storedDatahubJob.getFlowUrn().getFlowIdEntity()); diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/conf/SparkConfigParser.java b/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/conf/SparkConfigParser.java index d8da5d95935c9..c854861af2f81 100644 --- a/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/conf/SparkConfigParser.java +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/datahub/spark/conf/SparkConfigParser.java @@ -34,12 +34,16 @@ public class SparkConfigParser { public static final String COALESCE_KEY = "coalesce_jobs"; public static final String PATCH_ENABLED = "patch.enabled"; + public static final String DISABLE_SYMLINK_RESOLUTION = "disableSymlinkResolution"; public static final String STAGE_METADATA_COALESCING = "stage_metadata_coalescing"; public static final String STREAMING_JOB = "streaming_job"; public static final String STREAMING_HEARTBEAT = "streaming_heartbeat"; public static final String DATAHUB_FLOW_NAME = "flow_name"; public static final String DATASET_ENV_KEY = "metadata.dataset.env"; + public static final String DATASET_HIVE_PLATFORM_ALIAS = "metadata.dataset.hivePlatformAlias"; + public static final String DATASET_LOWERCASE_URNS = "metadata.dataset.lowerCaseUrns"; + public static final String DATASET_MATERIALIZE_KEY = "metadata.dataset.materialize"; public static final String DATASET_PLATFORM_INSTANCE_KEY = "metadata.dataset.platformInstance"; public static final String DATASET_INCLUDE_SCHEMA_METADATA = @@ -147,7 +151,10 @@ public static DatahubOpenlineageConfig sparkConfigToDatahubOpenlineageConf( } builder.platformInstance(SparkConfigParser.getPlatformInstance(sparkConfig)); builder.commonDatasetPlatformInstance(SparkConfigParser.getCommonPlatformInstance(sparkConfig)); + builder.hivePlatformAlias(SparkConfigParser.getHivePlatformAlias(sparkConfig)); builder.usePatch(SparkConfigParser.isPatchEnabled(sparkConfig)); + builder.disableSymlinkResolution(SparkConfigParser.isDisableSymlinkResolution(sparkConfig)); + builder.lowerCaseDatasetUrns(SparkConfigParser.isLowerCaseDatasetUrns(sparkConfig)); try { String parentJob = SparkConfigParser.getParentJobKey(sparkConfig); if (parentJob != null) { @@ -174,6 +181,12 @@ public static FabricType getCommonFabricType(Config datahubConfig) { return fabricType; } + public static String getHivePlatformAlias(Config datahubConfig) { + return datahubConfig.hasPath(DATASET_HIVE_PLATFORM_ALIAS) + ? datahubConfig.getString(DATASET_HIVE_PLATFORM_ALIAS) + : "hive"; + } + public static String getCommonPlatformInstance(Config datahubConfig) { return datahubConfig.hasPath(DATASET_PLATFORM_INSTANCE_KEY) ? datahubConfig.getString(DATASET_PLATFORM_INSTANCE_KEY) @@ -236,15 +249,18 @@ public static Map> getPathSpecListMap(Config datahubConfi pathSpecBuilder.alias(pathSpecKey); pathSpecBuilder.platform(key); if (datahubConfig.hasPath(aliasKey + ".env")) { - pathSpecBuilder.env(datahubConfig.getString(aliasKey + ".env")); + pathSpecBuilder.env(Optional.ofNullable(datahubConfig.getString(aliasKey + ".env"))); } - if (datahubConfig.hasPath(aliasKey + ".platformInstance")) { + if (datahubConfig.hasPath(aliasKey + "." + PLATFORM_INSTANCE_KEY)) { pathSpecBuilder.platformInstance( - Optional.ofNullable(datahubConfig.getString(aliasKey + ".platformInstance"))); + Optional.ofNullable( + datahubConfig.getString(aliasKey + "." + PLATFORM_INSTANCE_KEY))); + } + if (datahubConfig.hasPath(aliasKey + "." + PATH_SPEC_LIST_KEY)) { + pathSpecBuilder.pathSpecList( + Arrays.asList( + datahubConfig.getString(aliasKey + "." + PATH_SPEC_LIST_KEY).split(","))); } - pathSpecBuilder.pathSpecList( - Arrays.asList(datahubConfig.getString(aliasKey + "." + pathSpecKey).split(","))); - platformSpecs.add(pathSpecBuilder.build()); } pathSpecMap.put(key, platformSpecs); @@ -254,8 +270,8 @@ public static Map> getPathSpecListMap(Config datahubConfi } public static String getPlatformInstance(Config pathSpecConfig) { - return pathSpecConfig.hasPath(PLATFORM_INSTANCE_KEY) - ? pathSpecConfig.getString(PLATFORM_INSTANCE_KEY) + return pathSpecConfig.hasPath(PIPELINE_PLATFORM_INSTANCE_KEY) + ? pathSpecConfig.getString(PIPELINE_PLATFORM_INSTANCE_KEY) : null; } @@ -307,11 +323,19 @@ public static boolean isCoalesceEnabled(Config datahubConfig) { public static boolean isPatchEnabled(Config datahubConfig) { if (!datahubConfig.hasPath(PATCH_ENABLED)) { - return true; + return false; } return datahubConfig.hasPath(PATCH_ENABLED) && datahubConfig.getBoolean(PATCH_ENABLED); } + public static boolean isDisableSymlinkResolution(Config datahubConfig) { + if (!datahubConfig.hasPath(DISABLE_SYMLINK_RESOLUTION)) { + return false; + } + return datahubConfig.hasPath(DISABLE_SYMLINK_RESOLUTION) + && datahubConfig.getBoolean(DISABLE_SYMLINK_RESOLUTION); + } + public static boolean isEmitCoalescePeriodically(Config datahubConfig) { if (!datahubConfig.hasPath(STAGE_METADATA_COALESCING)) { // if databricks tags are present and stage_metadata_coalescing is not present, then default @@ -323,4 +347,9 @@ public static boolean isEmitCoalescePeriodically(Config datahubConfig) { return datahubConfig.hasPath(STAGE_METADATA_COALESCING) && datahubConfig.getBoolean(STAGE_METADATA_COALESCING); } + + public static boolean isLowerCaseDatasetUrns(Config datahubConfig) { + return datahubConfig.hasPath(DATASET_LOWERCASE_URNS) + && datahubConfig.getBoolean(DATASET_LOWERCASE_URNS); + } } diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/lifecycle/plan/InsertIntoHadoopFsRelationVisitor.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/lifecycle/plan/InsertIntoHadoopFsRelationVisitor.java new file mode 100644 index 0000000000000..1e5bc72967e68 --- /dev/null +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/lifecycle/plan/InsertIntoHadoopFsRelationVisitor.java @@ -0,0 +1,79 @@ +/* +/* Copyright 2018-2024 contributors to the OpenLineage project +/* SPDX-License-Identifier: Apache-2.0 +*/ + +package io.openlineage.spark.agent.lifecycle.plan; + +import io.openlineage.client.OpenLineage; +import io.openlineage.client.utils.DatasetIdentifier; +import io.openlineage.spark.agent.util.PathUtils; +import io.openlineage.spark.api.OpenLineageContext; +import io.openlineage.spark.api.QueryPlanVisitor; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.catalyst.catalog.CatalogTable; +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; +import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand; +import scala.Option; + +/** + * {@link LogicalPlan} visitor that matches an {@link InsertIntoHadoopFsRelationCommand} and + * extracts the output {@link OpenLineage.Dataset} being written. + */ +public class InsertIntoHadoopFsRelationVisitor + extends QueryPlanVisitor { + + public InsertIntoHadoopFsRelationVisitor(OpenLineageContext context) { + super(context); + } + + @Override + public List apply(LogicalPlan x) { + InsertIntoHadoopFsRelationCommand command = (InsertIntoHadoopFsRelationCommand) x; + + Option catalogTable = command.catalogTable(); + OpenLineage.OutputDataset outputDataset; + + if (catalogTable.isEmpty()) { + DatasetIdentifier di = PathUtils.fromURI(command.outputPath().toUri(), "file"); + if (SaveMode.Overwrite == command.mode()) { + outputDataset = + outputDataset() + .getDataset( + di, + command.query().schema(), + OpenLineage.LifecycleStateChangeDatasetFacet.LifecycleStateChange.OVERWRITE); + } else { + outputDataset = outputDataset().getDataset(di, command.query().schema()); + } + return Collections.singletonList(outputDataset); + } else { + if (SaveMode.Overwrite == command.mode()) { + return Collections.singletonList( + outputDataset() + .getDataset( + PathUtils.fromCatalogTable(catalogTable.get()), + catalogTable.get().schema(), + OpenLineage.LifecycleStateChangeDatasetFacet.LifecycleStateChange.CREATE)); + } else { + return Collections.singletonList( + outputDataset() + .getDataset( + PathUtils.fromCatalogTable(catalogTable.get()), catalogTable.get().schema())); + } + } + } + + @Override + public Optional jobNameSuffix(InsertIntoHadoopFsRelationCommand command) { + if (command.catalogTable().isEmpty()) { + DatasetIdentifier di = PathUtils.fromURI(command.outputPath().toUri(), "file"); + return Optional.of(trimPath(di.getName())); + } + return Optional.of( + trimPath(PathUtils.fromCatalogTable(command.catalogTable().get()).getName())); + } +} diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/RddPathUtils.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/RddPathUtils.java new file mode 100644 index 0000000000000..62005bf15f850 --- /dev/null +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/RddPathUtils.java @@ -0,0 +1,162 @@ +/* +/* Copyright 2018-2024 contributors to the OpenLineage project +/* SPDX-License-Identifier: Apache-2.0 +*/ + +package io.openlineage.spark.agent.util; + +import java.util.Arrays; +import java.util.Objects; +import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.spark.package$; +import org.apache.spark.rdd.HadoopRDD; +import org.apache.spark.rdd.MapPartitionsRDD; +import org.apache.spark.rdd.ParallelCollectionRDD; +import org.apache.spark.rdd.RDD; +import org.apache.spark.sql.execution.datasources.FileScanRDD; +import scala.Tuple2; +import scala.collection.immutable.Seq; + +/** Utility class to extract paths from RDD nodes. */ +@Slf4j +public class RddPathUtils { + + public static Stream findRDDPaths(RDD rdd) { + return Stream.of( + new HadoopRDDExtractor(), + new FileScanRDDExtractor(), + new MapPartitionsRDDExtractor(), + new ParallelCollectionRDDExtractor()) + .filter(e -> e.isDefinedAt(rdd)) + .findFirst() + .orElse(new UnknownRDDExtractor()) + .extract(rdd) + .filter(p -> p != null); + } + + static class UnknownRDDExtractor implements RddPathExtractor { + @Override + public boolean isDefinedAt(Object rdd) { + return true; + } + + @Override + public Stream extract(RDD rdd) { + // Change to debug to silence error + log.debug("Unknown RDD class {}", rdd); + return Stream.empty(); + } + } + + static class HadoopRDDExtractor implements RddPathExtractor { + @Override + public boolean isDefinedAt(Object rdd) { + return rdd instanceof HadoopRDD; + } + + @Override + public Stream extract(HadoopRDD rdd) { + org.apache.hadoop.fs.Path[] inputPaths = FileInputFormat.getInputPaths(rdd.getJobConf()); + Configuration hadoopConf = rdd.getConf(); + return Arrays.stream(inputPaths).map(p -> PlanUtils.getDirectoryPath(p, hadoopConf)); + } + } + + static class MapPartitionsRDDExtractor implements RddPathExtractor { + + @Override + public boolean isDefinedAt(Object rdd) { + return rdd instanceof MapPartitionsRDD; + } + + @Override + public Stream extract(MapPartitionsRDD rdd) { + return findRDDPaths(rdd.prev()); + } + } + + static class FileScanRDDExtractor implements RddPathExtractor { + @Override + public boolean isDefinedAt(Object rdd) { + return rdd instanceof FileScanRDD; + } + + @Override + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") + public Stream extract(FileScanRDD rdd) { + return ScalaConversionUtils.fromSeq(rdd.filePartitions()).stream() + .flatMap(fp -> Arrays.stream(fp.files())) + .map( + f -> { + if ("3.4".compareTo(package$.MODULE$.SPARK_VERSION()) <= 0) { + // filePath returns SparkPath for Spark 3.4 + return ReflectionUtils.tryExecuteMethod(f, "filePath") + .map(o -> ReflectionUtils.tryExecuteMethod(o, "toPath")) + .map(o -> (Path) o.get()) + .get() + .getParent(); + } else { + return parentOf(f.filePath()); + } + }); + } + } + + static class ParallelCollectionRDDExtractor implements RddPathExtractor { + @Override + public boolean isDefinedAt(Object rdd) { + return rdd instanceof ParallelCollectionRDD; + } + + @Override + public Stream extract(ParallelCollectionRDD rdd) { + try { + Object data = FieldUtils.readField(rdd, "data", true); + log.debug("ParallelCollectionRDD data: {}", data); + if (data instanceof Seq) { + return ScalaConversionUtils.fromSeq((Seq) data).stream() + .map( + el -> { + Path path = null; + if (el instanceof Tuple2) { + // we're able to extract path + path = parentOf(((Tuple2) el)._1.toString()); + log.debug("Found input {}", path); + } else { + // Change to debug to silence error + log.debug("unable to extract Path from {}", el.getClass().getCanonicalName()); + } + return path; + }) + .filter(Objects::nonNull); + } else { + // Changed to debug to silence error + log.debug("Cannot extract path from ParallelCollectionRDD {}", data); + } + } catch (IllegalAccessException | IllegalArgumentException e) { + // Changed to debug to silence error + log.debug("Cannot read data field from ParallelCollectionRDD {}", rdd); + } + return Stream.empty(); + } + } + + private static Path parentOf(String path) { + try { + return new Path(path).getParent(); + } catch (Exception e) { + return null; + } + } + + interface RddPathExtractor { + boolean isDefinedAt(Object rdd); + + Stream extract(T rdd); + } +} diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/RemovePathPatternUtils.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/RemovePathPatternUtils.java index c44dacf8ff3be..841298ab0e037 100644 --- a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/RemovePathPatternUtils.java +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/util/RemovePathPatternUtils.java @@ -18,6 +18,7 @@ import java.net.URISyntaxException; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -72,7 +73,7 @@ public static List removeOutputsPathPattern( .map( dataset -> { String newName = removePathPattern(dataset.getName()); - if (newName != dataset.getName()) { + if (!Objects.equals(newName, dataset.getName())) { return context .getOpenLineage() .newOutputDatasetBuilder() @@ -95,7 +96,7 @@ public static List removeInputsPathPattern( .map( dataset -> { String newName = removePathPattern(dataset.getName()); - if (newName != dataset.getName()) { + if (!Objects.equals(newName, dataset.getName())) { return context .getOpenLineage() .newInputDatasetBuilder() @@ -112,8 +113,8 @@ public static List removeInputsPathPattern( } private static Optional getPattern(OpenLineageContext context) { - return Optional.ofNullable(context.getSparkContext()) - .map(sparkContext -> sparkContext.conf()) + return Optional.of(context.getSparkContext()) + .map(sparkContext -> sparkContext.get().conf()) .filter(conf -> conf.contains(SPARK_OPENLINEAGE_DATASET_REMOVE_PATH_PATTERN)) .map(conf -> conf.get(SPARK_OPENLINEAGE_DATASET_REMOVE_PATH_PATTERN)) .map(pattern -> Pattern.compile(pattern)); diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/Constants.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/Constants.java new file mode 100644 index 0000000000000..717525e4a9aaf --- /dev/null +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/Constants.java @@ -0,0 +1,9 @@ +package io.openlineage.spark.agent.vendor.redshift; + +public class Constants { + public static final String REDSHIFT_CLASS_NAME = + "io.github.spark_redshift_community.spark.redshift.RedshiftRelation"; + + public static final String REDSHIFT_PROVIDER_CLASS_NAME = + "io.github.spark_redshift_community.spark.redshift.DefaultSource"; +} diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/RedshiftVendor.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/RedshiftVendor.java new file mode 100644 index 0000000000000..6f0fceb9c4c4a --- /dev/null +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/RedshiftVendor.java @@ -0,0 +1,56 @@ +package io.openlineage.spark.agent.vendor.redshift; + +import static io.openlineage.spark.agent.vendor.redshift.Constants.*; + +import io.openlineage.spark.agent.lifecycle.VisitorFactory; +import io.openlineage.spark.agent.vendor.redshift.lifecycle.RedshiftRelationVisitor; +import io.openlineage.spark.agent.vendor.redshift.lifecycle.plan.RedshiftEventHandlerFactory; +import io.openlineage.spark.agent.vendor.snowflake.lifecycle.SnowflakeVisitorFactory; +import io.openlineage.spark.api.OpenLineageEventHandlerFactory; +import io.openlineage.spark.api.Vendor; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class RedshiftVendor implements Vendor { + + public static boolean hasRedshiftClasses() { + /* + Checking the Redshift class with both + SnowflakeRelationVisitor.class.getClassLoader.loadClass and + Thread.currentThread().getContextClassLoader().loadClass. The first checks if the class is + present on the classpath, and the second one is a catchall which captures if the class has + been installed. This is relevant for Azure Databricks where jars can be installed and + accessible to the user, even if they are not present on the classpath. + */ + try { + RedshiftRelationVisitor.class.getClassLoader().loadClass(REDSHIFT_PROVIDER_CLASS_NAME); + return true; + } catch (Exception e) { + // swallow - we don't care + } + try { + Thread.currentThread().getContextClassLoader().loadClass(REDSHIFT_PROVIDER_CLASS_NAME); + return true; + } catch (Exception e) { + // swallow - we don't care + } + return false; + } + + @Override + public boolean isVendorAvailable() { + log.info("Checking if Redshift classes are available"); + return hasRedshiftClasses(); + } + + @Override + public Optional getVisitorFactory() { + return Optional.of(new SnowflakeVisitorFactory()); + } + + @Override + public Optional getEventHandlerFactory() { + return Optional.of(new RedshiftEventHandlerFactory()); + } +} diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/lifecycle/RedshiftDataset.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/lifecycle/RedshiftDataset.java new file mode 100644 index 0000000000000..0e7fd4ce942bd --- /dev/null +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/lifecycle/RedshiftDataset.java @@ -0,0 +1,72 @@ +package io.openlineage.spark.agent.vendor.redshift.lifecycle; + +import io.openlineage.client.OpenLineage; +import io.openlineage.spark.agent.util.SqlUtils; +import io.openlineage.spark.api.DatasetFactory; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Slf4j +public class RedshiftDataset { + public static final String REDSHIFT_PREFIX = "redshift://"; + + private static final Logger logger = LoggerFactory.getLogger(RedshiftDataset.class); + public static final String DEFAULT_SCHEMA = "public"; + + public static List getDatasets( + DatasetFactory factory, + String url, + Optional dbtable, + Optional query, + StructType schema) + throws URISyntaxException { + + URI jdbcUrl = + new URI( + REDSHIFT_PREFIX + + url.replace("jdbc:redshift:iam://", "").replace("jdbc:redshift://", "")); + String db = jdbcUrl.getPath().substring(1); // remove leading slash + final String namespace = + jdbcUrl.getScheme() + "://" + jdbcUrl.getHost() + ":" + jdbcUrl.getPort(); + + final String tableName; + // https://github.com/databricks/spark-redshift?tab=readme-ov-file + // > Specify one of the following options for the table data to be read: + // > - `dbtable`: The name of the table to be read. All columns and records are retrieved + // > (i.e. it is equivalent to SELECT * FROM db_table). + // > - `query`: The exact query (SELECT statement) to run. + // If dbtable is null it will be replaced with the string `complex` and it means the query + // option was used. + // An improvement could be put the query string in the `DatasetFacets` + if (dbtable.isPresent()) { + tableName = dbtable.get(); + String[] splits = tableName.split("\\."); + String table = tableName; + if (splits.length == 1) { + table = String.format("%s.%s.%s", db, DEFAULT_SCHEMA, tableName); + } else if (splits.length == 2) { + table = String.format("%s.%s", db, tableName); + } else if (splits.length == 3) { + table = tableName; + } else { + logger.warn("Redshift getDataset: tableName: {} is not in the expected format", tableName); + return Collections.emptyList(); + } + + return Collections.singletonList(factory.getDataset(table, namespace, schema)); + } else if (query.isPresent()) { + return SqlUtils.getDatasets(factory, query.get(), "redshift", namespace, db, DEFAULT_SCHEMA); + } else { + logger.warn( + "Unable to discover Redshift table property - neither \"dbtable\" nor \"query\" option present"); + } + return Collections.emptyList(); + } +} diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/lifecycle/RedshiftRelationVisitor.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/lifecycle/RedshiftRelationVisitor.java new file mode 100644 index 0000000000000..02eb2d1745558 --- /dev/null +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/lifecycle/RedshiftRelationVisitor.java @@ -0,0 +1,66 @@ +package io.openlineage.spark.agent.vendor.redshift.lifecycle; + +import io.github.spark_redshift_community.spark.redshift.Parameters; +import io.github.spark_redshift_community.spark.redshift.RedshiftRelation; +import io.github.spark_redshift_community.spark.redshift.TableName; +import io.openlineage.client.OpenLineage; +import io.openlineage.spark.agent.util.ScalaConversionUtils; +import io.openlineage.spark.api.DatasetFactory; +import io.openlineage.spark.api.OpenLineageContext; +import io.openlineage.spark.api.QueryPlanVisitor; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; +import org.apache.spark.sql.execution.datasources.LogicalRelation; +import org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand; + +/** + * {@link LogicalPlan} visitor that matches {@link SaveIntoDataSourceCommand}s that use a {@link + * RedshiftRelation}. This function extracts a {@link OpenLineage.Dataset} from the Redshift table + * referenced by the relation. + */ +@Slf4j +public class RedshiftRelationVisitor + extends QueryPlanVisitor { + private static final String REDSHIFT_NAMESPACE = "redshift"; + private static final String REDSHIFT_CLASS_NAME = + "io.github.spark_redshift_community.spark.redshift.RedshiftRelation"; + private final DatasetFactory factory; + + public RedshiftRelationVisitor(@NonNull OpenLineageContext context, DatasetFactory factory) { + super(context); + this.factory = factory; + log.info("RedshiftRelationVisitor created"); + } + + @Override + public List apply(LogicalPlan x) { + RedshiftRelation relation = (RedshiftRelation) ((LogicalRelation) x).relation(); + Parameters.MergedParameters params = relation.params(); + Optional dbtable = + (Optional) + ScalaConversionUtils.asJavaOptional(params.table().map(TableName::toString)); + Optional query = ScalaConversionUtils.asJavaOptional(params.query()); + return Collections.singletonList( + factory.getDataset(dbtable.orElse(""), REDSHIFT_NAMESPACE, relation.schema())); + } + + protected boolean isRedshiftClass(LogicalPlan plan) { + try { + Class c = Thread.currentThread().getContextClassLoader().loadClass(REDSHIFT_CLASS_NAME); + return (plan instanceof LogicalRelation + && c.isAssignableFrom(((LogicalRelation) plan).relation().getClass())); + } catch (Exception e) { + // swallow - not a snowflake class + } + return false; + } + + @Override + public boolean isDefinedAt(LogicalPlan plan) { + return isRedshiftClass(plan); + } +} diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/lifecycle/RedshiftVisitorFactory.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/lifecycle/RedshiftVisitorFactory.java new file mode 100644 index 0000000000000..1003c863227b8 --- /dev/null +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/lifecycle/RedshiftVisitorFactory.java @@ -0,0 +1,26 @@ +package io.openlineage.spark.agent.vendor.redshift.lifecycle; + +import io.openlineage.client.OpenLineage; +import io.openlineage.spark.agent.lifecycle.VisitorFactory; +import io.openlineage.spark.api.DatasetFactory; +import io.openlineage.spark.api.OpenLineageContext; +import java.util.Collections; +import java.util.List; +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; +import scala.PartialFunction; + +public class RedshiftVisitorFactory implements VisitorFactory { + @Override + public List>> getInputVisitors( + OpenLineageContext context) { + DatasetFactory factory = DatasetFactory.input(context); + return Collections.singletonList(new RedshiftRelationVisitor<>(context, factory)); + } + + @Override + public List>> getOutputVisitors( + OpenLineageContext context) { + DatasetFactory factory = DatasetFactory.output(context); + return Collections.singletonList(new RedshiftRelationVisitor<>(context, factory)); + } +} diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/lifecycle/plan/RedshiftEventHandlerFactory.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/lifecycle/plan/RedshiftEventHandlerFactory.java new file mode 100644 index 0000000000000..4cd1ba996fe88 --- /dev/null +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/lifecycle/plan/RedshiftEventHandlerFactory.java @@ -0,0 +1,20 @@ +package io.openlineage.spark.agent.vendor.redshift.lifecycle.plan; + +import io.openlineage.client.OpenLineage; +import io.openlineage.spark.api.OpenLineageContext; +import io.openlineage.spark.api.OpenLineageEventHandlerFactory; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import scala.PartialFunction; + +public class RedshiftEventHandlerFactory implements OpenLineageEventHandlerFactory { + @Override + public Collection>> + createOutputDatasetBuilder(OpenLineageContext context) { + // The right function will be determined at runtime by using type checking based on the correct + // Spark LogicalPlan + return Collections.singleton( + (PartialFunction) new RedshiftSaveIntoDataSourceCommandBuilder(context)); + } +} diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/lifecycle/plan/RedshiftSaveIntoDataSourceCommandBuilder.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/lifecycle/plan/RedshiftSaveIntoDataSourceCommandBuilder.java new file mode 100644 index 0000000000000..e484458d40aeb --- /dev/null +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/agent/vendor/redshift/lifecycle/plan/RedshiftSaveIntoDataSourceCommandBuilder.java @@ -0,0 +1,80 @@ +package io.openlineage.spark.agent.vendor.redshift.lifecycle.plan; + +import static io.openlineage.spark.agent.vendor.redshift.RedshiftVendor.hasRedshiftClasses; + +import io.openlineage.client.OpenLineage; +import io.openlineage.spark.agent.util.PlanUtils; +import io.openlineage.spark.agent.util.ScalaConversionUtils; +import io.openlineage.spark.agent.vendor.redshift.lifecycle.RedshiftDataset; +import io.openlineage.spark.api.AbstractQueryPlanDatasetBuilder; +import io.openlineage.spark.api.OpenLineageContext; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.scheduler.SparkListenerEvent; +import org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand; +import org.apache.spark.sql.sources.CreatableRelationProvider; +import org.apache.spark.sql.types.StructType; + +@Slf4j +public class RedshiftSaveIntoDataSourceCommandBuilder + extends AbstractQueryPlanDatasetBuilder< + SparkListenerEvent, SaveIntoDataSourceCommand, OpenLineage.OutputDataset> { + + public RedshiftSaveIntoDataSourceCommandBuilder(OpenLineageContext context) { + super(context, false); + } + + @Override + public List apply(SaveIntoDataSourceCommand command) { + if (isRedshiftSource(command.dataSource())) { + // Called from SaveIntoDataSourceCommandVisitor on Snowflake write operations. + Map options = ScalaConversionUtils.fromMap(command.options()); + log.info("Redshift SaveIntoDataSourceCommand options: {}", options); + Optional dbtable = Optional.ofNullable(options.get("dbtable")); + Optional query = Optional.ofNullable(options.get("query")); + String url = options.get("url"); + + try { + return + // Similar to Kafka, Snowflake also has some special handling. So we use the method + // below for extracting the dataset from Snowflake write operations. + RedshiftDataset.getDatasets( + outputDataset(), url, dbtable, query, getSchema(command) + // command.schema() doesn't seem to contain the schema when tested with Azure + // Snowflake, + // so we use the helper to extract it from the logical plan. + ); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } else { + return Collections.emptyList(); + } + } + + public static boolean isRedshiftSource(CreatableRelationProvider provider) { + return hasRedshiftClasses(); // && provider instanceof DefaultSource; + } + + /** + * Taken from {@link + * io.openlineage.spark.agent.lifecycle.plan.SaveIntoDataSourceCommandVisitor#getSchema(org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand)} + * + * @param command + * @return + */ + private StructType getSchema(SaveIntoDataSourceCommand command) { + StructType schema = command.schema(); + if ((schema == null || schema.fields() == null || schema.fields().length == 0) + && command.query() != null + && command.query().output() != null) { + // get schema from logical plan's output + schema = PlanUtils.toStructType(ScalaConversionUtils.fromSeq(command.query().output())); + } + return schema; + } +} diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/api/Vendors.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/api/Vendors.java new file mode 100644 index 0000000000000..967935cb40468 --- /dev/null +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/api/Vendors.java @@ -0,0 +1,80 @@ +/* +/* Copyright 2018-2024 contributors to the OpenLineage project +/* SPDX-License-Identifier: Apache-2.0 +*/ + +package io.openlineage.spark.api; + +import io.openlineage.spark.agent.lifecycle.VisitorFactory; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public interface Vendors { + + @SuppressWarnings("PMD.AvoidFieldNameMatchingTypeName") + List VENDORS = + Arrays.asList( + // Add vendor classes here + "io.openlineage.spark.agent.vendor.snowflake.SnowflakeVendor", + // This is the only chance we have to add the RedshiftVendor to the list of vendors + "io.openlineage.spark.agent.vendor.redshift.RedshiftVendor"); + + static Vendors getVendors() { + return getVendors(Collections.emptyList()); + } + + static Vendors getVendors(List additionalVendors) { + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + + List vendors = + Stream.concat(VENDORS.stream(), additionalVendors.stream()) + .map( + vendorClassName -> { + try { + Class vendor = cl.loadClass(vendorClassName); + return (Vendor) vendor.newInstance(); + } catch (ClassNotFoundException + | InstantiationException + | IllegalAccessException e) { + return null; + } + }) + .filter(Objects::nonNull) + .filter(Vendor::isVendorAvailable) + .collect(Collectors.toList()); + // The main reason to avoid using the service loader and use static loading with the class name + // is to prevent potential missing loading caused by missing META-INF/services files. + // This can happen if the user packages the OpenLineage dependency in an Uber-jar without proper + // services file configuration + // The implementation with the ClassLoader and the list of vendor class names increase the + // coupling between the vendor + // and the app + // https://github.com/OpenLineage/OpenLineage/issues/1860 + // ServiceLoader serviceLoader = ServiceLoader.load(Vendor.class); + return new VendorsImpl(vendors); + } + + static Vendors empty() { + return new Vendors() { + + @Override + public Collection getVisitorFactories() { + return Collections.emptyList(); + } + + @Override + public Collection getEventHandlerFactories() { + return Collections.emptyList(); + } + }; + } + + Collection getVisitorFactories(); + + Collection getEventHandlerFactories(); +} diff --git a/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/api/VendorsImpl.java b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/api/VendorsImpl.java new file mode 100644 index 0000000000000..66db4cf4f4e43 --- /dev/null +++ b/metadata-integration/java/spark-lineage-beta/src/main/java/io/openlineage/spark/api/VendorsImpl.java @@ -0,0 +1,42 @@ +/* +/* Copyright 2018-2024 contributors to the OpenLineage project +/* SPDX-License-Identifier: Apache-2.0 +*/ + +package io.openlineage.spark.api; + +import io.openlineage.spark.agent.lifecycle.VisitorFactory; +import io.openlineage.spark.agent.vendor.redshift.RedshiftVendor; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class VendorsImpl implements Vendors { + private final List vendors; + + public VendorsImpl(List vendors) { + this.vendors = vendors; + } + + @Override + public Collection getVisitorFactories() { + vendors.add(new RedshiftVendor()); + return vendors.stream() + .map(Vendor::getVisitorFactory) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + + @Override + public Collection getEventHandlerFactories() { + return vendors.stream() + .map(Vendor::getEventHandlerFactory) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } +} diff --git a/metadata-integration/java/spark-lineage-beta/src/test/java/datahub/spark/HdfsPathDatasetTest.java b/metadata-integration/java/spark-lineage-beta/src/test/java/datahub/spark/HdfsPathDatasetTest.java index bed4c197f9691..9d7637c6742b8 100644 --- a/metadata-integration/java/spark-lineage-beta/src/test/java/datahub/spark/HdfsPathDatasetTest.java +++ b/metadata-integration/java/spark-lineage-beta/src/test/java/datahub/spark/HdfsPathDatasetTest.java @@ -205,7 +205,7 @@ public void testPathSpecListPlatformInstance() HdfsPathDataset.create( new URI("s3a://my-bucket/foo/tests/bar.avro"), sparkLineageConf.getOpenLineageConf()); Assert.assertEquals( - "urn:li:dataset:(urn:li:dataPlatform:s3,instance.my-bucket/foo/tests,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:s3,my-bucket/foo/tests,PROD)", dataset.urn().toString()); } diff --git a/metadata-integration/java/spark-lineage-beta/src/test/java/datahub/spark/SparkStreamingEventToDatahubTest.java b/metadata-integration/java/spark-lineage-beta/src/test/java/datahub/spark/OpenLineageEventToDatahubTest.java similarity index 65% rename from metadata-integration/java/spark-lineage-beta/src/test/java/datahub/spark/SparkStreamingEventToDatahubTest.java rename to metadata-integration/java/spark-lineage-beta/src/test/java/datahub/spark/OpenLineageEventToDatahubTest.java index 71c43cea408e6..b42c87618d2b6 100644 --- a/metadata-integration/java/spark-lineage-beta/src/test/java/datahub/spark/SparkStreamingEventToDatahubTest.java +++ b/metadata-integration/java/spark-lineage-beta/src/test/java/datahub/spark/OpenLineageEventToDatahubTest.java @@ -13,12 +13,14 @@ import io.datahubproject.openlineage.converter.OpenLineageToDataHub; import io.datahubproject.openlineage.dataset.DatahubDataset; import io.datahubproject.openlineage.dataset.DatahubJob; +import io.datahubproject.openlineage.dataset.PathSpec; import io.openlineage.client.OpenLineage; import io.openlineage.client.OpenLineageClientUtils; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.List; import java.util.Optional; import java.util.stream.Stream; import junit.framework.TestCase; @@ -26,7 +28,7 @@ import org.apache.commons.lang3.tuple.Triple; import org.junit.Assert; -public class SparkStreamingEventToDatahubTest extends TestCase { +public class OpenLineageEventToDatahubTest extends TestCase { public void testGenerateUrnFromStreamingDescriptionFile() throws URISyntaxException { Config datahubConfig = ConfigFactory.parseMap( @@ -483,4 +485,273 @@ public void testProcessOlEventWithSetDatasetFabricType() throws URISyntaxExcepti dataset.getUrn().toString()); } } + + public void testProcessGlueOlEvent() throws URISyntaxException, IOException { + DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder = + DatahubOpenlineageConfig.builder(); + builder.fabricType(FabricType.DEV); + + String olEvent = + IOUtils.toString( + this.getClass().getResourceAsStream("/ol_events/sample_glue.json"), + StandardCharsets.UTF_8); + + OpenLineage.RunEvent runEvent = OpenLineageClientUtils.runEventFromJson(olEvent); + DatahubJob datahubJob = OpenLineageToDataHub.convertRunEventToJob(runEvent, builder.build()); + + assertNotNull(datahubJob); + + for (DatahubDataset dataset : datahubJob.getInSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:glue,my_glue_database.my_glue_table,DEV)", + dataset.getUrn().toString()); + } + for (DatahubDataset dataset : datahubJob.getOutSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:hive,my_glue_database.my_output_glue_table,DEV)", + dataset.getUrn().toString()); + } + } + + public void testProcessGlueOlEventSymlinkDisabled() throws URISyntaxException, IOException { + DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder = + DatahubOpenlineageConfig.builder(); + builder.fabricType(FabricType.DEV); + builder.disableSymlinkResolution(true); + + String olEvent = + IOUtils.toString( + this.getClass().getResourceAsStream("/ol_events/sample_glue.json"), + StandardCharsets.UTF_8); + + OpenLineage.RunEvent runEvent = OpenLineageClientUtils.runEventFromJson(olEvent); + DatahubJob datahubJob = OpenLineageToDataHub.convertRunEventToJob(runEvent, builder.build()); + + assertNotNull(datahubJob); + + for (DatahubDataset dataset : datahubJob.getInSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:s3,my-bucket-test/sample_data/input_data.parquet,DEV)", + dataset.getUrn().toString()); + } + for (DatahubDataset dataset : datahubJob.getOutSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:s3,my-bucket-test/sample_data/output_data.parquet,DEV)", + dataset.getUrn().toString()); + } + } + + public void testProcessGlueOlEventWithHiveAlias() throws URISyntaxException, IOException { + DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder = + DatahubOpenlineageConfig.builder(); + builder.fabricType(FabricType.DEV); + builder.hivePlatformAlias("glue"); + + String olEvent = + IOUtils.toString( + this.getClass().getResourceAsStream("/ol_events/sample_glue.json"), + StandardCharsets.UTF_8); + + OpenLineage.RunEvent runEvent = OpenLineageClientUtils.runEventFromJson(olEvent); + DatahubJob datahubJob = OpenLineageToDataHub.convertRunEventToJob(runEvent, builder.build()); + + assertNotNull(datahubJob); + + for (DatahubDataset dataset : datahubJob.getInSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:glue,my_glue_database.my_glue_table,DEV)", + dataset.getUrn().toString()); + } + for (DatahubDataset dataset : datahubJob.getOutSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:glue,my_glue_database.my_output_glue_table,DEV)", + dataset.getUrn().toString()); + } + } + + public void testProcessRedshiftOutput() throws URISyntaxException, IOException { + DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder = + DatahubOpenlineageConfig.builder(); + builder.fabricType(FabricType.DEV); + builder.hivePlatformAlias("glue"); + builder.materializeDataset(true); + builder.includeSchemaMetadata(true); + + String olEvent = + IOUtils.toString( + this.getClass().getResourceAsStream("/ol_events/redshift_lineage_spark.json"), + StandardCharsets.UTF_8); + + OpenLineage.RunEvent runEvent = OpenLineageClientUtils.runEventFromJson(olEvent); + DatahubJob datahubJob = OpenLineageToDataHub.convertRunEventToJob(runEvent, builder.build()); + + assertNotNull(datahubJob); + + for (DatahubDataset dataset : datahubJob.getInSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:mysql,datahub.metadata_aspect_v2,DEV)", + dataset.getUrn().toString()); + } + for (DatahubDataset dataset : datahubJob.getOutSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.spark_redshift_load_test,DEV)", + dataset.getUrn().toString()); + assertEquals( + dataset.getSchemaMetadata().getPlatform().toString(), "urn:li:dataPlatform:redshift"); + } + } + + public void testProcessRedshiftOutputWithPlatformInstance() + throws URISyntaxException, IOException { + DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder = + DatahubOpenlineageConfig.builder(); + builder.fabricType(FabricType.DEV); + builder.hivePlatformAlias("glue"); + builder.materializeDataset(true); + builder.includeSchemaMetadata(true); + builder.commonDatasetPlatformInstance("my-platform-instance"); + + String olEvent = + IOUtils.toString( + this.getClass().getResourceAsStream("/ol_events/redshift_lineage_spark.json"), + StandardCharsets.UTF_8); + + OpenLineage.RunEvent runEvent = OpenLineageClientUtils.runEventFromJson(olEvent); + DatahubJob datahubJob = OpenLineageToDataHub.convertRunEventToJob(runEvent, builder.build()); + + assertNotNull(datahubJob); + + for (DatahubDataset dataset : datahubJob.getInSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:mysql,my-platform-instance.datahub.metadata_aspect_v2,DEV)", + dataset.getUrn().toString()); + } + for (DatahubDataset dataset : datahubJob.getOutSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:redshift,my-platform-instance.dev.public.spark_redshift_load_test,DEV)", + dataset.getUrn().toString()); + assertEquals( + dataset.getSchemaMetadata().getPlatform().toString(), "urn:li:dataPlatform:redshift"); + } + } + + public void testProcessRedshiftOutputWithPlatformSpecificPlatformInstance() + throws URISyntaxException, IOException { + DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder = + DatahubOpenlineageConfig.builder(); + builder.fabricType(FabricType.DEV); + builder.hivePlatformAlias("glue"); + builder.materializeDataset(true); + builder.includeSchemaMetadata(true); + builder.pathSpecs( + new HashMap>() { + { + put( + "redshift", + List.of( + PathSpec.builder() + .platform("redshift") + .platformInstance(Optional.of("my-platform-instance")) + .build())); + } + }); + + String olEvent = + IOUtils.toString( + this.getClass().getResourceAsStream("/ol_events/redshift_lineage_spark.json"), + StandardCharsets.UTF_8); + + OpenLineage.RunEvent runEvent = OpenLineageClientUtils.runEventFromJson(olEvent); + DatahubJob datahubJob = OpenLineageToDataHub.convertRunEventToJob(runEvent, builder.build()); + + assertNotNull(datahubJob); + + for (DatahubDataset dataset : datahubJob.getInSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:mysql,datahub.metadata_aspect_v2,DEV)", + dataset.getUrn().toString()); + } + for (DatahubDataset dataset : datahubJob.getOutSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:redshift,my-platform-instance.dev.public.spark_redshift_load_test,DEV)", + dataset.getUrn().toString()); + assertEquals( + dataset.getSchemaMetadata().getPlatform().toString(), "urn:li:dataPlatform:redshift"); + } + } + + public void testProcessRedshiftOutputWithPlatformSpecificEnv() + throws URISyntaxException, IOException { + DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder = + DatahubOpenlineageConfig.builder(); + builder.fabricType(FabricType.DEV); + builder.hivePlatformAlias("glue"); + builder.materializeDataset(true); + builder.includeSchemaMetadata(true); + builder.pathSpecs( + new HashMap>() { + { + put( + "redshift", + List.of(PathSpec.builder().platform("redshift").env(Optional.of("PROD")).build())); + } + }); + + String olEvent = + IOUtils.toString( + this.getClass().getResourceAsStream("/ol_events/redshift_lineage_spark.json"), + StandardCharsets.UTF_8); + + OpenLineage.RunEvent runEvent = OpenLineageClientUtils.runEventFromJson(olEvent); + DatahubJob datahubJob = OpenLineageToDataHub.convertRunEventToJob(runEvent, builder.build()); + + assertNotNull(datahubJob); + + for (DatahubDataset dataset : datahubJob.getInSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:mysql,datahub.metadata_aspect_v2,DEV)", + dataset.getUrn().toString()); + } + for (DatahubDataset dataset : datahubJob.getOutSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.spark_redshift_load_test,PROD)", + dataset.getUrn().toString()); + assertEquals( + dataset.getSchemaMetadata().getPlatform().toString(), "urn:li:dataPlatform:redshift"); + } + } + + public void testProcessRedshiftOutputLowercasedUrns() throws URISyntaxException, IOException { + DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder = + DatahubOpenlineageConfig.builder(); + builder.fabricType(FabricType.DEV); + builder.hivePlatformAlias("glue"); + builder.materializeDataset(true); + builder.includeSchemaMetadata(true); + builder.lowerCaseDatasetUrns(true); + + String olEvent = + IOUtils.toString( + this.getClass() + .getResourceAsStream("/ol_events/redshift_mixed_case_lineage_spark.json"), + StandardCharsets.UTF_8); + + OpenLineage.RunEvent runEvent = OpenLineageClientUtils.runEventFromJson(olEvent); + DatahubJob datahubJob = OpenLineageToDataHub.convertRunEventToJob(runEvent, builder.build()); + + assertNotNull(datahubJob); + + for (DatahubDataset dataset : datahubJob.getInSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:mysql,datahub.metadata_aspect_v2,DEV)", + dataset.getUrn().toString()); + } + for (DatahubDataset dataset : datahubJob.getOutSet()) { + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.spark_redshift_load_test,DEV)", + dataset.getUrn().toString()); + assertEquals( + dataset.getSchemaMetadata().getPlatform().toString(), "urn:li:dataPlatform:redshift"); + } + } } diff --git a/metadata-integration/java/spark-lineage-beta/src/test/resources/ol_events/redshift_lineage_spark.json b/metadata-integration/java/spark-lineage-beta/src/test/resources/ol_events/redshift_lineage_spark.json new file mode 100644 index 0000000000000..3b11b28207636 --- /dev/null +++ b/metadata-integration/java/spark-lineage-beta/src/test/resources/ol_events/redshift_lineage_spark.json @@ -0,0 +1,147 @@ +{ + "eventTime": "2024-06-18T06:52:21.64Z", + "producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent", + "eventType": "COMPLETE", + "run": { + "runId": "01902a1e-371a-7dbf-8098-2337d441e8dc", + "facets": { + "parent": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/ParentRunFacet.json#/$defs/ParentRunFacet", + "run": { + "runId": "01902a1e-0b05-750e-b38d-439998f7a853" + }, + "job": { + "namespace": "default", + "name": "jdbc_test_demo" + } + }, + "processing_engine": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/ProcessingEngineRunFacet.json#/$defs/ProcessingEngineRunFacet", + "version": "3.3.4", + "name": "spark" + }, + "environment-properties": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", + "environment-properties": {} + }, + "spark_properties": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", + "properties": { + "spark.master": "local[*]", + "spark.app.name": "JdbcTest-Demo" + } + } + } + }, + "job": { + "namespace": "default", + "name": "jdbc_test_demo.execute_save_into_data_source_command.spark_redshift_load_test", + "facets": { + "jobType": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/2-0-2/JobTypeJobFacet.json#/$defs/JobTypeJobFacet", + "processingType": "BATCH", + "integration": "SPARK", + "jobType": "SQL_JOB" + } + } + }, + "inputs": [ + { + "namespace": "mysql://localhost:3306", + "name": "datahub.metadata_aspect_v2", + "facets": { + "dataSource": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", + "name": "mysql://localhost:3306", + "uri": "mysql://localhost:3306" + }, + "schema": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", + "fields": [ + { + "name": "urn", + "type": "string" + }, + { + "name": "aspect", + "type": "string" + }, + { + "name": "version", + "type": "long" + }, + { + "name": "metadata", + "type": "string" + }, + { + "name": "systemmetadata", + "type": "string" + }, + { + "name": "createdon", + "type": "timestamp" + }, + { + "name": "createdby", + "type": "string" + }, + { + "name": "createdfor", + "type": "string" + } + ] + } + }, + "inputFacets": {} + } + ], + "outputs": [ + { + "namespace": "redshift://my-redshift-cluster.us-west-2.redshift.amazonaws.com:5439", + "name": "dev.public.spark_redshift_load_test", + "facets": { + "dataSource": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", + "name": "redshift://my-redshift-cluster.us-west-2.redshift.amazonaws.com:5439", + "uri": "redshift://my-redshift-cluster.us-west-2.redshift.amazonaws.com:5439" + }, + "schema": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", + "fields": [ + { + "name": "urn", + "type": "string" + } + ] + }, + "columnLineage": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-2/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet", + "fields": { + "urn": { + "inputFields": [ + { + "namespace": "mysql://localhost:3306", + "name": "datahub.metadata_aspect_v2", + "field": "urn" + } + ] + } + } + } + }, + "outputFacets": {} + } + ] +} \ No newline at end of file diff --git a/metadata-integration/java/spark-lineage-beta/src/test/resources/ol_events/redshift_mixed_case_lineage_spark.json b/metadata-integration/java/spark-lineage-beta/src/test/resources/ol_events/redshift_mixed_case_lineage_spark.json new file mode 100644 index 0000000000000..692eebe64bb5c --- /dev/null +++ b/metadata-integration/java/spark-lineage-beta/src/test/resources/ol_events/redshift_mixed_case_lineage_spark.json @@ -0,0 +1,147 @@ +{ + "eventTime": "2024-06-18T06:52:21.64Z", + "producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent", + "eventType": "COMPLETE", + "run": { + "runId": "01902a1e-371a-7dbf-8098-2337d441e8dc", + "facets": { + "parent": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/ParentRunFacet.json#/$defs/ParentRunFacet", + "run": { + "runId": "01902a1e-0b05-750e-b38d-439998f7a853" + }, + "job": { + "namespace": "default", + "name": "jdbc_test_demo" + } + }, + "processing_engine": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/ProcessingEngineRunFacet.json#/$defs/ProcessingEngineRunFacet", + "version": "3.3.4", + "name": "spark" + }, + "environment-properties": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", + "environment-properties": {} + }, + "spark_properties": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", + "properties": { + "spark.master": "local[*]", + "spark.app.name": "JdbcTest-Demo" + } + } + } + }, + "job": { + "namespace": "default", + "name": "jdbc_test_demo.execute_save_into_data_source_command.spark_redshift_load_test", + "facets": { + "jobType": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/2-0-2/JobTypeJobFacet.json#/$defs/JobTypeJobFacet", + "processingType": "BATCH", + "integration": "SPARK", + "jobType": "SQL_JOB" + } + } + }, + "inputs": [ + { + "namespace": "mysql://localhost:3306", + "name": "DataHub.Metadata_Aspect_V2", + "facets": { + "dataSource": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", + "name": "mysql://localhost:3306", + "uri": "mysql://localhost:3306" + }, + "schema": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", + "fields": [ + { + "name": "urn", + "type": "string" + }, + { + "name": "aspect", + "type": "string" + }, + { + "name": "version", + "type": "long" + }, + { + "name": "metadata", + "type": "string" + }, + { + "name": "systemmetadata", + "type": "string" + }, + { + "name": "createdon", + "type": "timestamp" + }, + { + "name": "createdby", + "type": "string" + }, + { + "name": "createdfor", + "type": "string" + } + ] + } + }, + "inputFacets": {} + } + ], + "outputs": [ + { + "namespace": "redshift://my-redshift-cluster.us-west-2.redshift.amazonaws.com:5439", + "name": "DEV.PuBliC.SparK_RedshifT_Load_Test", + "facets": { + "dataSource": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", + "name": "redshift://my-redshift-cluster.us-west-2.redshift.amazonaws.com:5439", + "uri": "redshift://my-redshift-cluster.us-west-2.redshift.amazonaws.com:5439" + }, + "schema": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", + "fields": [ + { + "name": "urn", + "type": "string" + } + ] + }, + "columnLineage": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.16.0/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-2/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet", + "fields": { + "urn": { + "inputFields": [ + { + "namespace": "mysql://localhost:3306", + "name": "datahub.metadata_aspect_v2", + "field": "urn" + } + ] + } + } + } + }, + "outputFacets": {} + } + ] +} \ No newline at end of file diff --git a/metadata-integration/java/spark-lineage-beta/src/test/resources/ol_events/sample_glue.json b/metadata-integration/java/spark-lineage-beta/src/test/resources/ol_events/sample_glue.json new file mode 100644 index 0000000000000..a1959ba37c0f3 --- /dev/null +++ b/metadata-integration/java/spark-lineage-beta/src/test/resources/ol_events/sample_glue.json @@ -0,0 +1,168 @@ +{ + "eventTime": "2024-05-31T17:01:26.465Z", + "producer": "https://github.com/OpenLineage/OpenLineage/tree/1.13.1/integration/spark", + "schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent", + "eventType": "START", + "run": { + "runId": "3ad2a5ec-1c8b-4bda-84f4-1492758af65c", + "facets": { + "parent": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.13.1/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/ParentRunFacet.json#/$defs/ParentRunFacet", + "run": { + "runId": "f03077f2-c077-4472-987e-b89b1c741c86" + }, + "job": { + "namespace": "default", + "name": "simple_app_parquet_with_persist_without_coalesce_s3_demo" + } + }, + "processing_engine": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.13.1/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/ProcessingEngineRunFacet.json#/$defs/ProcessingEngineRunFacet", + "version": "3.3.0-amzn-1", + "name": "spark" + }, + "environment-properties": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.13.1/integration/spark", + "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", + "environment-properties": {} + }, + "spark_properties": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.13.1/integration/spark", + "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", + "properties": { + "spark.master": "jes", + "spark.app.name": "SimpleAppParquetWithPersistWithoutCoalesceS3-Demo" + } + }, + "spark_version": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.13.1/integration/spark", + "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", + "spark-version": "3.3.0-amzn-1" + } + } + }, + "job": { + "namespace": "default", + "name": "simple_app_parquet_with_persist_without_coalesce_s3_demo.execute_insert_into_hadoop_fs_relation_command.sample_data_output", + "facets": { + "jobType": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.13.1/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/2-0-2/JobTypeJobFacet.json#/$defs/JobTypeJobFacet", + "processingType": "BATCH", + "integration": "SPARK", + "jobType": "JOB" + } + } + }, + "inputs": [ + { + "namespace": "s3://my-bucket-test", + "name": "/sample_data/input_data.parquet", + "facets": { + "dataSource": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.13.1/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", + "name": "s3://my-bucket-test", + "uri": "s3://my-bucket-test" + }, + "schema": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.13.1/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", + "fields": [ + { + "name": "field_1", + "type": "integer" + }, + { + "name": "field_2", + "type": "string" + } + ] + }, + "symlinks": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.13.1/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet", + "identifiers": [ + { + "namespace": "aws:glue:us-west-2:123456789012", + "name": "my_glue_database.my_glue_table", + "type": "TABLE" + } + ] + } + }, + "inputFacets": {} + } + ], + "outputs": [ + { + "namespace": "s3://my-bucket-test", + "name": "mydata _output", + "facets": { + "dataSource": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.13.1/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", + "name": "s3://my-bucket-test", + "uri": "s3://my-bucket-test" + }, + "schema": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.13.1/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", + "fields": [ + { + "name": "field1", + "type": "long" + }, + { + "name": "field2", + "type": "string" + } + ] + }, + "columnLineage": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.13.1/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-2/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet", + "fields": { + "field1": { + "inputFields": [ + { + "namespace": "s3://my-bucket-test", + "name": "/output/field1.parquet", + "field": "field1" + } + ] + }, + "field2": { + "inputFields": [ + { + "namespace": "s3://my-bucket-test", + "name": "/output/field2.parquet", + "field": "field2" + } + ] + } + } + }, + "symlinks": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.13.1/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet", + "identifiers": [ + { + "namespace": "s3://my-bucket-test/my-warehouse/", + "name": "my_glue_database.my_output_glue_table", + "type": "TABLE" + } + ] + }, + "lifecycleStateChange": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.13.1/integration/spark", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/LifecycleStateChangeDatasetFacet.json#/$defs/LifecycleStateChangeDatasetFacet", + "lifecycleStateChange": "OVERWRITE" + } + }, + "outputFacets": {} + } + ] +} \ No newline at end of file diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index 5bd73c844b380..6666e33544688 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -92,6 +92,8 @@ dependencies { testImplementation 'ch.qos.logback:logback-classic:1.4.7' testImplementation 'net.datafaker:datafaker:1.9.0' + testImplementation(testFixtures(project(":entity-registry"))) + testAnnotationProcessor externalDependency.lombok constraints { diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImpl.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImpl.java index 30e9251982f10..6f45a36d1daf4 100644 --- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImpl.java +++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImpl.java @@ -4,6 +4,7 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.StringMap; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.SystemAspect; @@ -23,7 +24,10 @@ import com.linkedin.mxe.SystemMetadata; import java.io.IOException; import java.sql.Timestamp; +import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.Builder; @@ -69,7 +73,7 @@ public static ChangeItemImpl fromPatch( @Nonnull private final RecordTemplate recordTemplate; - @Nonnull private final SystemMetadata systemMetadata; + @Nonnull private SystemMetadata systemMetadata; @Nonnull private final AuditStamp auditStamp; @@ -80,19 +84,44 @@ public static ChangeItemImpl fromPatch( @Nonnull private final AspectSpec aspectSpec; @Setter @Nullable private SystemAspect previousSystemAspect; - @Setter private long nextAspectVersion; + private long nextAspectVersion; + private final Map headers; + + @Override + public void setNextAspectVersion(long nextAspectVersion) { + this.nextAspectVersion = nextAspectVersion; + try { + this.systemMetadata = new SystemMetadata(getSystemMetadata().copy().data()); + } catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + this.systemMetadata.setVersion(String.valueOf(nextAspectVersion + 1)); + } @Nonnull @Override - public SystemAspect getSystemAspect(@Nullable Long version) { + public SystemAspect getSystemAspect(@Nullable Long nextAspectVersion) { EntityAspect entityAspect = new EntityAspect(); entityAspect.setAspect(getAspectName()); entityAspect.setMetadata(EntityApiUtils.toJsonAspect(getRecordTemplate())); entityAspect.setUrn(getUrn().toString()); - entityAspect.setVersion(version == null ? getNextAspectVersion() : version); + entityAspect.setVersion(nextAspectVersion == null ? getNextAspectVersion() : nextAspectVersion); entityAspect.setCreatedOn(new Timestamp(getAuditStamp().getTime())); entityAspect.setCreatedBy(getAuditStamp().getActor().toString()); - entityAspect.setSystemMetadata(EntityApiUtils.toJsonAspect(getSystemMetadata())); + if (nextAspectVersion != null) { + // Apply version to system metadata (copy to ensure we don't pollute shared systemMetadata + // objects across aspects) + SystemMetadata updatedSystemMetadata = null; + try { + updatedSystemMetadata = new SystemMetadata(getSystemMetadata().copy().data()); + } catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + updatedSystemMetadata.setVersion(String.valueOf(nextAspectVersion + 1)); + entityAspect.setSystemMetadata(EntityApiUtils.toJsonAspect(updatedSystemMetadata)); + } else { + entityAspect.setSystemMetadata(EntityApiUtils.toJsonAspect(getSystemMetadata())); + } return EntityAspect.EntitySystemAspect.builder() .build(getEntitySpec(), getAspectSpec(), entityAspect); } @@ -112,10 +141,24 @@ public MetadataChangeProposal getMetadataChangeProposal() { mcp.setEntityKeyAspect( GenericRecordUtils.serializeAspect( EntityKeyUtils.convertUrnToEntityKey(getUrn(), entitySpec.getKeyAspectSpec()))); + if (!headers.isEmpty()) { + mcp.setHeaders(new StringMap(headers)); + } return mcp; } } + @Override + public Map getHeaders() { + return Optional.ofNullable(metadataChangeProposal) + .filter(MetadataChangeProposal::hasHeaders) + .map( + mcp -> + mcp.getHeaders().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) + .orElse(headers); + } + public static class ChangeItemImplBuilder { // Ensure use of other builders @@ -133,6 +176,11 @@ public ChangeItemImpl build(AspectRetriever aspectRetriever) { // Apply change type default this.changeType = validateOrDefaultChangeType(changeType); + // Apply empty headers + if (this.headers == null) { + this.headers = Map.of(); + } + ValidationApiUtils.validateUrn(aspectRetriever.getEntityRegistry(), this.urn); log.debug("entity type = {}", this.urn.getEntityType()); @@ -156,7 +204,8 @@ public ChangeItemImpl build(AspectRetriever aspectRetriever) { this.entitySpec, this.aspectSpec, this.previousSystemAspect, - this.nextAspectVersion); + this.nextAspectVersion, + this.headers); } public static ChangeItemImpl build( diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/PatchItemImpl.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/PatchItemImpl.java index f4473c8db3148..43a7d00248a22 100644 --- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/PatchItemImpl.java +++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/PatchItemImpl.java @@ -10,6 +10,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; +import com.linkedin.data.ByteString; import com.linkedin.data.template.RecordTemplate; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.AspectRetriever; @@ -21,7 +22,9 @@ import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.metadata.utils.SystemMetadataUtils; +import com.linkedin.mxe.GenericAspect; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.SystemMetadata; import jakarta.json.Json; @@ -78,6 +81,29 @@ public RecordTemplate getRecordTemplate() { return null; } + @Nonnull + public MetadataChangeProposal getMetadataChangeProposal() { + if (metadataChangeProposal != null) { + return metadataChangeProposal; + } else { + GenericAspect genericAspect = new GenericAspect(); + genericAspect.setContentType("application/json"); + genericAspect.setValue(ByteString.copyString(getPatch().toString(), StandardCharsets.UTF_8)); + + final MetadataChangeProposal mcp = new MetadataChangeProposal(); + mcp.setEntityUrn(getUrn()); + mcp.setChangeType(getChangeType()); + mcp.setEntityType(getEntitySpec().getName()); + mcp.setAspectName(getAspectName()); + mcp.setAspect(genericAspect); + mcp.setSystemMetadata(getSystemMetadata()); + mcp.setEntityKeyAspect( + GenericRecordUtils.serializeAspect( + EntityKeyUtils.convertUrnToEntityKey(getUrn(), entitySpec.getKeyAspectSpec()))); + return mcp; + } + } + public ChangeItemImpl applyPatch(RecordTemplate recordTemplate, AspectRetriever aspectRetriever) { ChangeItemImpl.ChangeItemImplBuilder builder = ChangeItemImpl.builder() diff --git a/metadata-io/src/main/java/com/linkedin/metadata/aspect/utils/DefaultAspectsUtil.java b/metadata-io/src/main/java/com/linkedin/metadata/aspect/utils/DefaultAspectsUtil.java index 21bac3cbb0e61..a4b2e991b6e1e 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/aspect/utils/DefaultAspectsUtil.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/aspect/utils/DefaultAspectsUtil.java @@ -12,6 +12,7 @@ import com.linkedin.common.BrowsePathsV2; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.SetMode; import com.linkedin.data.template.StringArray; import com.linkedin.dataplatform.DataPlatformInfo; import com.linkedin.entity.EntityResponse; @@ -29,6 +30,7 @@ import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.GenericAspect; import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.mxe.SystemMetadata; import com.linkedin.util.Pair; import io.datahubproject.metadata.context.OperationContext; import java.util.Collection; @@ -132,7 +134,7 @@ public static List getAdditionalChanges( getProposalFromAspectForDefault( entry.getKey(), entry.getValue(), entityKeyAspect, templateItem), templateItem.getAuditStamp(), - opContext.getRetrieverContext().get().getAspectRetriever())) + opContext.getAspectRetrieverOpt().get())) .filter(Objects::nonNull); }) .collect(Collectors.toList()); @@ -348,7 +350,14 @@ public static MetadataChangeProposal getProposalFromAspectForDefault( // Set fields determined from original if (templateItem.getSystemMetadata() != null) { - proposal.setSystemMetadata(templateItem.getSystemMetadata()); + SystemMetadata systemMetadata = null; + try { + systemMetadata = new SystemMetadata(templateItem.getSystemMetadata().copy().data()); + } catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + systemMetadata.setVersion(null, SetMode.REMOVE_IF_NULL); + proposal.setSystemMetadata(systemMetadata); } if (templateItem.getUrn() != null) { proposal.setEntityUrn(templateItem.getUrn()); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/client/EntityClientAspectRetriever.java b/metadata-io/src/main/java/com/linkedin/metadata/client/EntityClientAspectRetriever.java index 3ee382ab0d74f..bba8324d0c561 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/client/EntityClientAspectRetriever.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/client/EntityClientAspectRetriever.java @@ -4,12 +4,15 @@ import com.linkedin.entity.Aspect; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.aspect.CachingAspectRetriever; +import com.linkedin.metadata.aspect.SystemAspect; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.r2.RemoteInvocationException; import io.datahubproject.metadata.context.OperationContext; import java.net.URISyntaxException; +import java.util.Collection; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.Builder; @@ -52,4 +55,25 @@ public Map> getLatestAspectObjects( } } } + + @Nonnull + @Override + public Map> getLatestSystemAspects( + Map> urnAspectNames) { + if (urnAspectNames.isEmpty()) { + return Map.of(); + } else { + try { + // TODO: This generates over-fetching if not all aspects are needed for each URN + return entityClient.getLatestSystemAspect( + systemOperationContext, + urnAspectNames.keySet(), + urnAspectNames.values().stream() + .flatMap(Collection::stream) + .collect(Collectors.toSet())); + } catch (RemoteInvocationException | URISyntaxException e) { + throw new RuntimeException(e); + } + } + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java index ec25a2fee76d5..337288ab59c60 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java @@ -112,7 +112,7 @@ public EntityResponse getV2( @Nonnull @Deprecated public Entity get(@Nonnull OperationContext opContext, @Nonnull final Urn urn) { - return entityService.getEntity(opContext, urn, ImmutableSet.of()); + return entityService.getEntity(opContext, urn, ImmutableSet.of(), true); } @Nonnull @@ -175,7 +175,7 @@ public Map batchGetVersionedV2( @Deprecated public Map batchGet( @Nonnull OperationContext opContext, @Nonnull final Set urns) { - return entityService.getEntities(opContext, urns, ImmutableSet.of()); + return entityService.getEntities(opContext, urns, ImmutableSet.of(), true); } /** diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceAspectRetriever.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceAspectRetriever.java index 5377220758534..626a1f72f5fb7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceAspectRetriever.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceAspectRetriever.java @@ -1,15 +1,19 @@ package com.linkedin.metadata.entity; import static com.linkedin.metadata.utils.GenericRecordUtils.entityResponseToAspectMap; +import static com.linkedin.metadata.utils.GenericRecordUtils.entityResponseToSystemAspectMap; import com.linkedin.common.urn.Urn; import com.linkedin.entity.Aspect; import com.linkedin.metadata.aspect.CachingAspectRetriever; +import com.linkedin.metadata.aspect.SystemAspect; import com.linkedin.metadata.models.registry.EntityRegistry; import io.datahubproject.metadata.context.OperationContext; import java.net.URISyntaxException; +import java.util.Collection; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.Builder; @@ -48,4 +52,30 @@ public Map> getLatestAspectObjects( } } } + + @Nonnull + @Override + public Map> getLatestSystemAspects( + Map> urnAspectNames) { + if (urnAspectNames.isEmpty()) { + return Map.of(); + } else { + String entityName = + urnAspectNames.keySet().stream().findFirst().map(Urn::getEntityType).get(); + try { + // TODO - This causes over-fetching if not all aspects are required for every URN + return entityResponseToSystemAspectMap( + entityService.getEntitiesV2( + systemOperationContext, + entityName, + urnAspectNames.keySet(), + urnAspectNames.values().stream() + .flatMap(Collection::stream) + .collect(Collectors.toSet())), + entityRegistry); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java index 353b83726611e..6f2f95b8b115a 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java @@ -10,6 +10,7 @@ import static com.linkedin.metadata.utils.PegasusUtils.constructMCL; import static com.linkedin.metadata.utils.PegasusUtils.getDataTemplateClassFromSchema; import static com.linkedin.metadata.utils.PegasusUtils.urnToEntityName; +import static com.linkedin.metadata.utils.metrics.ExceptionUtils.collectMetrics; import com.codahale.metrics.Timer; import com.datahub.util.RecordUtils; @@ -44,6 +45,7 @@ import com.linkedin.metadata.aspect.batch.AspectsBatch; import com.linkedin.metadata.aspect.batch.BatchItem; import com.linkedin.metadata.aspect.batch.ChangeMCP; +import com.linkedin.metadata.aspect.batch.MCLItem; import com.linkedin.metadata.aspect.batch.MCPItem; import com.linkedin.metadata.aspect.plugins.validation.ValidationExceptionCollection; import com.linkedin.metadata.aspect.utils.DefaultAspectsUtil; @@ -53,6 +55,7 @@ import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl; import com.linkedin.metadata.entity.ebean.batch.DeleteItemImpl; +import com.linkedin.metadata.entity.ebean.batch.MCLItemImpl; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesResult; import com.linkedin.metadata.entity.retention.BulkApplyRetentionArgs; @@ -100,6 +103,7 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.stream.StreamSupport; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.persistence.EntityNotFoundException; @@ -152,6 +156,7 @@ public class EntityServiceImpl implements EntityService { @Nullable @Getter private SearchIndicesService updateIndicesService; private final PreProcessHooks preProcessHooks; protected static final int MAX_KEYS_PER_QUERY = 500; + protected static final int MCP_SIDE_EFFECT_KAFKA_BATCH_SIZE = 500; private final Integer ebeanMaxTransactionRetry; private final boolean enableBrowseV2; @@ -210,7 +215,8 @@ public RecordTemplate getLatestAspect( public Map> getLatestAspects( @Nonnull OperationContext opContext, @Nonnull final Set urns, - @Nonnull final Set aspectNames) { + @Nonnull final Set aspectNames, + boolean alwaysIncludeKeyAspect) { Map batchGetResults = getLatestAspect(opContext, urns, aspectNames); @@ -223,15 +229,17 @@ public Map> getLatestAspects( urnToAspects.putIfAbsent(urn, new ArrayList<>()); } - // Add "key" aspects for each urn. TODO: Replace this with a materialized key aspect. - urnToAspects - .keySet() - .forEach( - key -> { - final RecordTemplate keyAspect = - EntityApiUtils.buildKeyAspect(opContext.getEntityRegistry(), key); - urnToAspects.get(key).add(keyAspect); - }); + if (alwaysIncludeKeyAspect) { + // Add "key" aspects for each urn. TODO: Replace this with a materialized key aspect. + urnToAspects + .keySet() + .forEach( + key -> { + final RecordTemplate keyAspect = + EntityApiUtils.buildKeyAspect(opContext.getEntityRegistry(), key); + urnToAspects.get(key).add(keyAspect); + }); + } List systemAspects = EntityUtils.toSystemAspects( @@ -328,9 +336,12 @@ public EntityResponse getEntityV2( @Nonnull OperationContext opContext, @Nonnull final String entityName, @Nonnull final Urn urn, - @Nonnull final Set aspectNames) + @Nonnull final Set aspectNames, + boolean alwaysIncludeKeyAspect) throws URISyntaxException { - return getEntitiesV2(opContext, entityName, Collections.singleton(urn), aspectNames).get(urn); + return getEntitiesV2( + opContext, entityName, Collections.singleton(urn), aspectNames, alwaysIncludeKeyAspect) + .get(urn); } /** @@ -348,9 +359,12 @@ public Map getEntitiesV2( @Nonnull OperationContext opContext, @Nonnull final String entityName, @Nonnull final Set urns, - @Nonnull final Set aspectNames) + @Nonnull final Set aspectNames, + boolean alwaysIncludeKeyAspect) throws URISyntaxException { - return getLatestEnvelopedAspects(opContext, urns, aspectNames).entrySet().stream() + return getLatestEnvelopedAspects(opContext, urns, aspectNames, alwaysIncludeKeyAspect) + .entrySet() + .stream() .collect( Collectors.toMap( Map.Entry::getKey, @@ -370,9 +384,13 @@ public Map getEntitiesV2( public Map getEntitiesVersionedV2( @Nonnull OperationContext opContext, @Nonnull final Set versionedUrns, - @Nonnull final Set aspectNames) + @Nonnull final Set aspectNames, + boolean alwaysIncludeKeyAspect) throws URISyntaxException { - return getVersionedEnvelopedAspects(opContext, versionedUrns, aspectNames).entrySet().stream() + return getVersionedEnvelopedAspects( + opContext, versionedUrns, aspectNames, alwaysIncludeKeyAspect) + .entrySet() + .stream() .collect( Collectors.toMap( Map.Entry::getKey, @@ -388,23 +406,105 @@ public Map getEntitiesVersionedV2( */ @Override public Map> getLatestEnvelopedAspects( - @Nonnull OperationContext opContext, @Nonnull Set urns, @Nonnull Set aspectNames) - throws URISyntaxException { + @Nonnull OperationContext opContext, + @Nonnull Set urns, + @Nonnull Set aspectNames, + boolean alwaysIncludeKeyAspect) { - final Set dbKeys = + return getEnvelopedVersionedAspects( + opContext, urns.stream() .map( urn -> - aspectNames.stream() - .map( - aspectName -> - new EntityAspectIdentifier( - urn.toString(), aspectName, ASPECT_LATEST_VERSION)) - .collect(Collectors.toList())) - .flatMap(List::stream) + Map.entry( + urn, + aspectNames.stream() + .map(aspectName -> Map.entry(aspectName, 0L)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)), + alwaysIncludeKeyAspect); + } + + @Override + public Map> getEnvelopedVersionedAspects( + @Nonnull OperationContext opContext, + @Nonnull Map> urnAspectVersions, + boolean alwaysIncludeKeyAspect) { + + // we will always need to fetch latest aspects in case the requested version is version 0 being + // requested with version != 0 + Map>> withLatest = + urnAspectVersions.entrySet().stream() + .map( + entry -> + Map.entry( + entry.getKey(), + entry.getValue().entrySet().stream() + .map( + aspectEntry -> + Map.entry( + aspectEntry.getKey(), + Stream.of(0L, aspectEntry.getValue()) + .collect(Collectors.toSet()))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + Map> latestResult = + getEnvelopedVersionedAspectsInternal(opContext, withLatest, alwaysIncludeKeyAspect); + + return latestResult.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + a -> + a.getValue().stream() + .filter( + v -> + matchVersion(v, urnAspectVersions.get(a.getKey()).get(v.getName()))) + .collect(Collectors.toList()))); + } + + private static boolean matchVersion( + @Nonnull EnvelopedAspect envelopedAspect, @Nullable Long expectedVersion) { + if (expectedVersion == null) { + return true; + } + if (Objects.equals(envelopedAspect.getVersion(GetMode.NULL), expectedVersion)) { + return true; + } + if (envelopedAspect.hasSystemMetadata() + && envelopedAspect.getSystemMetadata().hasVersion() + && envelopedAspect.getSystemMetadata().getVersion() != null) { + return Objects.equals( + Long.parseLong(envelopedAspect.getSystemMetadata().getVersion()), expectedVersion); + } + + return false; + } + + private Map> getEnvelopedVersionedAspectsInternal( + @Nonnull OperationContext opContext, + @Nonnull Map>> urnAspectVersions, + boolean alwaysIncludeKeyAspect) { + final Set dbKeys = + urnAspectVersions.entrySet().stream() + .flatMap( + entry -> { + Urn urn = entry.getKey(); + return entry.getValue().entrySet().stream() + .flatMap( + aspectNameVersion -> + aspectNameVersion.getValue().stream() + .map( + version -> + new EntityAspectIdentifier( + urn.toString(), + aspectNameVersion.getKey(), + version))); + }) .collect(Collectors.toSet()); - return getCorrespondingAspects(opContext, dbKeys, urns); + return getCorrespondingAspects(opContext, dbKeys, alwaysIncludeKeyAspect); } /** @@ -419,7 +519,8 @@ public Map> getLatestEnvelopedAspects( public Map> getVersionedEnvelopedAspects( @Nonnull OperationContext opContext, @Nonnull Set versionedUrns, - @Nonnull Set aspectNames) + @Nonnull Set aspectNames, + boolean alwaysIncludeKeyAspect) throws URISyntaxException { Map> urnAspectVersionMap = @@ -460,17 +561,16 @@ public Map> getVersionedEnvelopedAspects( .flatMap(List::stream) .collect(Collectors.toSet())); - return getCorrespondingAspects( - opContext, - dbKeys, - versionedUrns.stream() - .map(versionedUrn -> versionedUrn.getUrn().toString()) - .map(UrnUtils::getUrn) - .collect(Collectors.toSet())); + return getCorrespondingAspects(opContext, dbKeys, alwaysIncludeKeyAspect); } private Map> getCorrespondingAspects( - @Nonnull OperationContext opContext, Set dbKeys, Set urns) { + @Nonnull OperationContext opContext, + Set dbKeys, + boolean alwaysIncludeKeyAspect) { + + Set urns = + dbKeys.stream().map(dbKey -> UrnUtils.getUrn(dbKey.getUrn())).collect(Collectors.toSet()); final Map envelopedAspectMap = getEnvelopedAspects(opContext, dbKeys); @@ -487,11 +587,14 @@ private Map> getCorrespondingAspects( for (Urn urn : urns) { List aspects = urnToAspects.getOrDefault(urn.toString(), Collections.emptyList()); + EnvelopedAspect keyAspect = EntityUtils.getKeyEnvelopedAspect(urn, opContext.getEntityRegistry()); // Add key aspect if it does not exist in the returned aspects - if (aspects.isEmpty() - || aspects.stream().noneMatch(aspect -> keyAspect.getName().equals(aspect.getName()))) { + if (alwaysIncludeKeyAspect + && (aspects.isEmpty() + || aspects.stream() + .noneMatch(aspect -> keyAspect.getName().equals(aspect.getName())))) { result.put( urn, ImmutableList.builder().addAll(aspects).add(keyAspect).build()); } else { @@ -635,7 +738,7 @@ public List ingestAspects( .recordTemplate(pair.getValue()) .systemMetadata(systemMetadata) .auditStamp(auditStamp) - .build(opContext.getRetrieverContext().get().getAspectRetriever())) + .build(opContext.getAspectRetrieverOpt().get())) .collect(Collectors.toList()); return ingestAspects( opContext, @@ -672,9 +775,50 @@ public List ingestAspects( ingestAspectsToLocalDB(opContext, aspectsBatch, overwrite); List mclResults = emitMCL(opContext, ingestResults, emitMCL); + + processPostCommitMCLSideEffects( + opContext, + mclResults.stream() + .filter(result -> !result.isNoOp()) + .map(UpdateAspectResult::toMCL) + .collect(Collectors.toList())); + return mclResults; } + /** + * Process post-commit MCPSideEffects + * + * @param mcls mcls generated + */ + private void processPostCommitMCLSideEffects( + @Nonnull OperationContext opContext, List mcls) { + log.debug("Considering {} MCLs post commit side effects.", mcls.size()); + List batch = + mcls.stream() + .map(mcl -> MCLItemImpl.builder().build(mcl, opContext.getAspectRetrieverOpt().get())) + .collect(Collectors.toList()); + + Iterable> iterable = + () -> + Iterators.partition( + AspectsBatch.applyPostMCPSideEffects(batch, opContext.getRetrieverContext().get()) + .iterator(), + MCP_SIDE_EFFECT_KAFKA_BATCH_SIZE); + StreamSupport.stream(iterable.spliterator(), false) + .forEach( + sideEffects -> { + long count = + ingestProposalAsync( + AspectsBatchImpl.builder() + .items(sideEffects) + .retrieverContext(opContext.getRetrieverContext().get()) + .build()) + .count(); + log.info("Generated {} MCP SideEffects for async processing", count); + }); + } + /** * Checks whether there is an actual update to the aspect by applying the updateLambda If there is * an update, push the new version into the local DB. Otherwise, do not push the new version, but @@ -710,9 +854,9 @@ private List ingestAspectsToLocalDB( EntityUtils.toSystemAspects( opContext.getRetrieverContext().get(), aspectDao.getLatestAspects(urnAspects)); - // read #2 + // read #2 (potentially) final Map> nextVersions = - aspectDao.getNextVersions(urnAspects); + EntityUtils.calculateNextVersions(aspectDao, latestAspects, urnAspects); // 1. Convert patches to full upserts // 2. Run any entity/aspect level hooks @@ -727,10 +871,13 @@ private List ingestAspectsToLocalDB( EntityUtils.toSystemAspects( opContext.getRetrieverContext().get(), aspectDao.getLatestAspects(updatedItems.getFirst())); - Map> newNextVersions = - aspectDao.getNextVersions(updatedItems.getFirst()); // merge updatedLatestAspects = AspectsBatch.merge(latestAspects, newLatestAspects); + + Map> newNextVersions = + EntityUtils.calculateNextVersions( + aspectDao, updatedLatestAspects, updatedItems.getFirst()); + // merge updatedNextVersions = AspectsBatch.merge(nextVersions, newNextVersions); } else { updatedLatestAspects = latestAspects; @@ -767,16 +914,16 @@ private List ingestAspectsToLocalDB( }) .collect(Collectors.toList()); + // No changes, return + if (changeMCPs.isEmpty()) { + return Collections.emptyList(); + } + // do final pre-commit checks with previous aspect value ValidationExceptionCollection exceptions = AspectsBatch.validatePreCommit(changeMCPs, opContext.getRetrieverContext().get()); if (!exceptions.isEmpty()) { - throw new ValidationException(exceptions.toString()); - } - - // No changes, return - if (changeMCPs.isEmpty()) { - return Collections.emptyList(); + throw new ValidationException(collectMetrics(exceptions).toString()); } // Database Upsert results @@ -960,7 +1107,7 @@ public RecordTemplate ingestAspectIfNotPresent( .recordTemplate(newValue) .systemMetadata(systemMetadata) .auditStamp(auditStamp) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), opContext.getRetrieverContext().get()) .build(); List ingested = ingestAspects(opContext, aspectsBatch, true, false); @@ -1058,7 +1205,7 @@ private Stream ingestTimeseriesProposal( .recordTemplate( EntityApiUtils.buildKeyAspect( opContext.getEntityRegistry(), item.getUrn())) - .build(opContext.getRetrieverContext().get().getAspectRetriever())) + .build(opContext.getAspectRetrieverOpt().get())) .collect(Collectors.toList()); ingestProposalSync( @@ -1452,7 +1599,7 @@ private RestoreIndicesResult restoreIndices( .auditStamp(auditStamp) .systemMetadata(latestSystemMetadata) .recordTemplate(EntityApiUtils.buildKeyAspect(opContext.getEntityRegistry(), urn)) - .build(opContext.getRetrieverContext().get().getAspectRetriever())); + .build(opContext.getAspectRetrieverOpt().get())); Stream defaultAspectsResult = ingestProposalSync( opContext, @@ -1532,8 +1679,11 @@ public ListUrnsResult listUrns( public Entity getEntity( @Nonnull OperationContext opContext, @Nonnull final Urn urn, - @Nonnull final Set aspectNames) { - return getEntities(opContext, Collections.singleton(urn), aspectNames).values().stream() + @Nonnull final Set aspectNames, + boolean alwaysIncludeKeyAspect) { + return getEntities(opContext, Collections.singleton(urn), aspectNames, alwaysIncludeKeyAspect) + .values() + .stream() .findFirst() .orElse(null); } @@ -1552,12 +1702,15 @@ public Entity getEntity( public Map getEntities( @Nonnull OperationContext opContext, @Nonnull final Set urns, - @Nonnull Set aspectNames) { + @Nonnull Set aspectNames, + boolean alwaysIncludeKeyAspect) { log.debug("Invoked getEntities with urns {}, aspects {}", urns, aspectNames); if (urns.isEmpty()) { return Collections.emptyMap(); } - return getSnapshotUnions(opContext, urns, aspectNames).entrySet().stream() + return getSnapshotUnions(opContext, urns, aspectNames, alwaysIncludeKeyAspect) + .entrySet() + .stream() .collect( Collectors.toMap(Map.Entry::getKey, entry -> EntityUtils.toEntity(entry.getValue()))); } @@ -1708,8 +1861,11 @@ public void ingestEntity( protected Map getSnapshotUnions( @Nonnull OperationContext opContext, @Nonnull final Set urns, - @Nonnull final Set aspectNames) { - return getSnapshotRecords(opContext, urns, aspectNames).entrySet().stream() + @Nonnull final Set aspectNames, + boolean alwaysIncludeKeyAspect) { + return getSnapshotRecords(opContext, urns, aspectNames, alwaysIncludeKeyAspect) + .entrySet() + .stream() .collect( Collectors.toMap( Map.Entry::getKey, entry -> EntityUtils.toSnapshotUnion(entry.getValue()))); @@ -1719,8 +1875,11 @@ protected Map getSnapshotUnions( protected Map getSnapshotRecords( @Nonnull OperationContext opContext, @Nonnull final Set urns, - @Nonnull final Set aspectNames) { - return getLatestAspectUnions(opContext, urns, aspectNames).entrySet().stream() + @Nonnull final Set aspectNames, + boolean alwaysIncludeKeyAspect) { + return getLatestAspectUnions(opContext, urns, aspectNames, alwaysIncludeKeyAspect) + .entrySet() + .stream() .collect( Collectors.toMap( Map.Entry::getKey, @@ -1731,8 +1890,11 @@ protected Map getSnapshotRecords( protected Map> getLatestAspectUnions( @Nonnull OperationContext opContext, @Nonnull final Set urns, - @Nonnull final Set aspectNames) { - return this.getLatestAspects(opContext, urns, aspectNames).entrySet().stream() + @Nonnull final Set aspectNames, + boolean alwaysIncludeKeyAspect) { + return this.getLatestAspects(opContext, urns, aspectNames, alwaysIncludeKeyAspect) + .entrySet() + .stream() .collect( Collectors.toMap( Map.Entry::getKey, @@ -1768,7 +1930,7 @@ private void ingestSnapshotUnion( .recordTemplate(pair.getValue()) .auditStamp(auditStamp) .systemMetadata(systemMetadata) - .build(opContext.getRetrieverContext().get().getAspectRetriever())) + .build(opContext.getAspectRetrieverOpt().get())) .collect(Collectors.toList())) .build(); @@ -1832,6 +1994,7 @@ public RollbackRunResult rollbackWithConditions( Map conditions, boolean hardDelete) { List removedAspects = new ArrayList<>(); + List removedAspectResults = new ArrayList<>(); AtomicInteger rowsDeletedFromEntityDeletion = new AtomicInteger(0); List> futures = @@ -1839,7 +2002,7 @@ public RollbackRunResult rollbackWithConditions( .map( aspectToRemove -> { RollbackResult result = - deleteAspect( + deleteAspectWithoutMCL( opContext, aspectToRemove.getUrn(), aspectToRemove.getAspectName(), @@ -1860,6 +2023,7 @@ public RollbackRunResult rollbackWithConditions( rowsDeletedFromEntityDeletion.addAndGet(result.additionalRowsAffected); removedAspects.add(aspectToRemove); + removedAspectResults.add(result); return alwaysProduceMCLAsync( opContext, result.getUrn(), @@ -1890,12 +2054,14 @@ public RollbackRunResult rollbackWithConditions( } }); - return new RollbackRunResult(removedAspects, rowsDeletedFromEntityDeletion.get()); + return new RollbackRunResult( + removedAspects, rowsDeletedFromEntityDeletion.get(), removedAspectResults); } @Override public RollbackRunResult deleteUrn(@Nonnull OperationContext opContext, Urn urn) { List removedAspects = new ArrayList<>(); + List removedAspectResults = new ArrayList<>(); Integer rowsDeletedFromEntityDeletion = 0; final EntitySpec spec = @@ -1910,7 +2076,8 @@ public RollbackRunResult deleteUrn(@Nonnull OperationContext opContext, Urn urn) log.warn("Entity to delete does not exist. {}", urn.toString()); } if (latestKey == null || latestKey.getSystemMetadata() == null) { - return new RollbackRunResult(removedAspects, rowsDeletedFromEntityDeletion); + return new RollbackRunResult( + removedAspects, rowsDeletedFromEntityDeletion, removedAspectResults); } SystemMetadata latestKeySystemMetadata = @@ -1918,7 +2085,7 @@ public RollbackRunResult deleteUrn(@Nonnull OperationContext opContext, Urn urn) .map(SystemAspect::getSystemMetadata) .get(); RollbackResult result = - deleteAspect( + deleteAspectWithoutMCL( opContext, urn.toString(), keyAspectName, @@ -1935,6 +2102,7 @@ public RollbackRunResult deleteUrn(@Nonnull OperationContext opContext, Urn urn) rowsDeletedFromEntityDeletion = result.additionalRowsAffected; removedAspects.add(summary); + removedAspectResults.add(result); Future future = alwaysProduceMCLAsync( opContext, @@ -1960,7 +2128,8 @@ public RollbackRunResult deleteUrn(@Nonnull OperationContext opContext, Urn urn) } } - return new RollbackRunResult(removedAspects, rowsDeletedFromEntityDeletion); + return new RollbackRunResult( + removedAspects, rowsDeletedFromEntityDeletion, removedAspectResults); } @Override @@ -2001,7 +2170,7 @@ public Set exists( } else { // Additionally exclude status.removed == true Map> statusResult = - getLatestAspects(opContext, existing, Set.of(STATUS_ASPECT_NAME)); + getLatestAspects(opContext, existing, Set.of(STATUS_ASPECT_NAME), false); return existing.stream() .filter( urn -> @@ -2013,9 +2182,9 @@ public Set exists( } } + /** Does not emit MCL */ @Nullable - @Override - public RollbackResult deleteAspect( + private RollbackResult deleteAspectWithoutMCL( @Nonnull OperationContext opContext, String urn, String aspectName, @@ -2035,13 +2204,13 @@ public RollbackResult deleteAspect( .urn(entityUrn) .aspectName(aspectName) .auditStamp(auditStamp) - .build(opContext.getRetrieverContext().get().getAspectRetriever()); + .build(opContext.getAspectRetrieverOpt().get()); // Delete validation hooks ValidationExceptionCollection exceptions = AspectsBatch.validateProposed(List.of(deleteItem), opContext.getRetrieverContext().get()); if (!exceptions.isEmpty()) { - throw new ValidationException(exceptions.toString()); + throw new ValidationException(collectMetrics(exceptions).toString()); } final RollbackResult result = @@ -2115,7 +2284,7 @@ public RollbackResult deleteAspect( .collect(Collectors.toList()), opContext.getRetrieverContext().get()); if (!preCommitExceptions.isEmpty()) { - throw new ValidationException(preCommitExceptions.toString()); + throw new ValidationException(collectMetrics(preCommitExceptions).toString()); } // 5. Apply deletes and fix up latest row @@ -2203,6 +2372,10 @@ public RollbackResult deleteAspect( }, DEFAULT_MAX_TRANSACTION_RETRY); + if (result != null) { + processPostCommitMCLSideEffects(opContext, List.of(result.toMCL(auditStamp))); + } + return result; } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java index e542b10af4ddc..7842365ce429b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java @@ -34,7 +34,9 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; @@ -159,7 +161,7 @@ static EntityResponse toEntityResponse( * Prefer batched interfaces * * @param entityAspect optional entity aspect - * @param aspectRetriever + * @param retrieverContext * @return */ public static Optional toSystemAspect( @@ -175,7 +177,7 @@ public static Optional toSystemAspect( * translate that into our java classes * * @param rawAspects `Map>` - * @param aspectRetriever used for read mutations + * @param retrieverContext used for read mutations * @return the java map for the given database object map */ @Nonnull @@ -278,4 +280,73 @@ public static List toSystemAspects( return systemAspects; } + + /** + * Use the precalculated next version from system metadata if it exists, otherwise lookup the next + * version the normal way from the database + * + * @param aspectDao database access + * @param latestAspects aspect version 0 with system metadata + * @param urnAspects urn/aspects which we need next version information for + * @return map of the urn/aspect to the next aspect version + */ + public static Map> calculateNextVersions( + AspectDao aspectDao, + Map> latestAspects, + Map> urnAspects) { + Map> precalculatedVersions = + latestAspects.entrySet().stream() + .map( + entry -> + Map.entry( + entry.getKey(), convertSystemAspectToNextVersionMap(entry.getValue()))) + .filter(entry -> !entry.getValue().isEmpty()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + Map> missingAspectVersions = + urnAspects.entrySet().stream() + .flatMap( + entry -> + entry.getValue().stream() + .map(aspectName -> Pair.of(entry.getKey(), aspectName))) + .filter( + urnAspectName -> + !precalculatedVersions + .getOrDefault(urnAspectName.getKey(), Map.of()) + .containsKey(urnAspectName.getValue())) + .collect( + Collectors.groupingBy( + Pair::getKey, Collectors.mapping(Pair::getValue, Collectors.toSet()))); + Map> databaseVersions = + missingAspectVersions.isEmpty() + ? Map.of() + : aspectDao.getNextVersions(missingAspectVersions); + + // stitch back together the precalculated and database versions + return Stream.concat( + precalculatedVersions.entrySet().stream(), databaseVersions.entrySet().stream()) + .collect( + Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue, + (m1, m2) -> + Stream.concat(m1.entrySet().stream(), m2.entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))); + } + + /** + * Given a map of aspect name to system aspect, extract the next version if it exists + * + * @param aspectMap aspect name to system aspect map + * @return aspect name to next aspect version + */ + private static Map convertSystemAspectToNextVersionMap( + Map aspectMap) { + return aspectMap.entrySet().stream() + .filter(entry -> entry.getValue().getVersion() == 0) + .map(entry -> Map.entry(entry.getKey(), entry.getValue().getSystemMetadataVersion())) + .filter(entry -> entry.getValue().isPresent()) + .map(entry -> Map.entry(entry.getKey(), entry.getValue().get())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/dgraph/DgraphGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/dgraph/DgraphGraphService.java index fe9bcb0daba91..27b603244d3b3 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/dgraph/DgraphGraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/dgraph/DgraphGraphService.java @@ -765,9 +765,6 @@ private void removeIncomingEdgesFromNode( _dgraph.executeConsumer(client -> client.newTransaction().doRequest(request)); } - @Override - public void configure() {} - @Override public void clear() { log.debug("dropping Dgraph data"); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java index 00c19fd3835cf..e21c4e15b4fc9 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java @@ -37,7 +37,6 @@ import io.opentelemetry.extension.annotations.WithSpan; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -65,9 +64,18 @@ import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.search.SearchHit; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.bucket.filter.FilterAggregationBuilder; +import org.opensearch.search.aggregations.bucket.filter.ParsedFilter; +import org.opensearch.search.aggregations.bucket.terms.ParsedStringTerms; +import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.opensearch.search.aggregations.metrics.ParsedTopHits; +import org.opensearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.rescore.QueryRescorerBuilder; +import org.opensearch.search.sort.SortOrder; /** A search DAO for Elasticsearch backend. */ @Slf4j @@ -89,7 +97,13 @@ public class ESGraphQueryDAO { static final String UPDATED_ON = "updatedOn"; static final String UPDATED_ACTOR = "updatedActor"; static final String PROPERTIES = "properties"; + static final String SCORE_FIELD = "_score"; static final String UI = "UI"; + static final String FILTER_BY_SOURCE_RELATIONSHIP = "filter_by_source_relationship"; + static final String FILTER_BY_DESTINATION_RELATIONSHIP = "filter_by_destination_relationship"; + static final String GROUP_BY_SOURCE_AGG = "group_by_source"; + static final String GROUP_BY_DESTINATION_AGG = "group_by_destination"; + static final String TOP_DOCUMENTS_AGG = "top_documents"; @Nonnull public static void addFilterToQueryBuilder( @@ -118,15 +132,7 @@ private SearchResponse executeLineageSearchQuery( @Nonnull final QueryBuilder query, final int offset, final int count) { SearchRequest searchRequest = new SearchRequest(); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - - searchSourceBuilder.from(offset); - searchSourceBuilder.size(count); - - searchSourceBuilder.query(query); - if (graphQueryConfiguration.isBoostViaNodes()) { - addViaNodeBoostQuery(searchSourceBuilder); - } + SearchSourceBuilder searchSourceBuilder = sharedSourceBuilder(query, offset, count); searchRequest.source(searchSourceBuilder); @@ -141,24 +147,82 @@ private SearchResponse executeLineageSearchQuery( } } - private SearchResponse executeLineageSearchQuery( - @Nonnull final QueryBuilder query, - @Nullable Object[] sort, - @Nullable String pitId, - @Nonnull String keepAlive, - final int count) { - SearchRequest searchRequest = new SearchRequest(); - + private SearchSourceBuilder sharedSourceBuilder( + @Nonnull final QueryBuilder query, final int offset, final int count) { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - ESUtils.setSearchAfter(searchSourceBuilder, sort, pitId, keepAlive); + searchSourceBuilder.from(offset); searchSourceBuilder.size(count); + searchSourceBuilder.query(query); + if (graphQueryConfiguration.isBoostViaNodes()) { + addViaNodeBoostQuery(searchSourceBuilder); + } + return searchSourceBuilder; + } + private SearchResponse executeGroupByLineageSearchQuery( + @Nonnull final QueryBuilder query, + final int offset, + final int count, + final Set> validEdges) { + SearchRequest searchRequest = new SearchRequest(); + + SearchSourceBuilder searchSourceBuilder = sharedSourceBuilder(query, offset, 0); + + // We have to group by both Source AND Destination because edge types may go in different + // directions for lineage + // set up filters for each relationship type in the correct direction to limit buckets + BoolQueryBuilder sourceFilterQuery = QueryBuilders.boolQuery(); + sourceFilterQuery.minimumShouldMatch(1); + validEdges.stream() + .filter(pair -> RelationshipDirection.OUTGOING.equals(pair.getValue().getDirection())) + .forEach( + pair -> + sourceFilterQuery.should( + getAggregationFilter(pair, RelationshipDirection.OUTGOING))); + + BoolQueryBuilder destFilterQuery = QueryBuilders.boolQuery(); + destFilterQuery.minimumShouldMatch(1); + validEdges.stream() + .filter(pair -> RelationshipDirection.INCOMING.equals(pair.getValue().getDirection())) + .forEach( + pair -> + destFilterQuery.should(getAggregationFilter(pair, RelationshipDirection.INCOMING))); + + FilterAggregationBuilder sourceRelationshipTypeFilters = + AggregationBuilders.filter(FILTER_BY_SOURCE_RELATIONSHIP, sourceFilterQuery); + FilterAggregationBuilder destRelationshipTypeFilters = + AggregationBuilders.filter(FILTER_BY_DESTINATION_RELATIONSHIP, destFilterQuery); + TermsAggregationBuilder sourceAgg = + AggregationBuilders.terms(GROUP_BY_SOURCE_AGG) + .field(SOURCE + ".urn") + .size( + graphQueryConfiguration + .getBatchSize()); // Number of buckets can be up to batch size per query for + // each + + TermsAggregationBuilder destAgg = + AggregationBuilders.terms(GROUP_BY_DESTINATION_AGG) + .field(DESTINATION + ".urn") + .size(graphQueryConfiguration.getBatchSize()); + + TopHitsAggregationBuilder topHitsAgg = + AggregationBuilders.topHits(TOP_DOCUMENTS_AGG) + .size(count) + .sort(SCORE_FIELD, SortOrder.DESC); + sourceAgg.subAggregation(topHitsAgg); + destAgg.subAggregation(topHitsAgg); + + sourceRelationshipTypeFilters.subAggregation(sourceAgg); + destRelationshipTypeFilters.subAggregation(destAgg); + searchSourceBuilder.aggregation(sourceRelationshipTypeFilters); + searchSourceBuilder.aggregation(destRelationshipTypeFilters); searchRequest.source(searchSourceBuilder); searchRequest.indices(indexConvention.getIndexName(INDEX_NAME)); - try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "esQuery").time()) { + try (Timer.Context ignored = + MetricUtils.timer(this.getClass(), "esLineageGroupByQuery").time()) { MetricUtils.counter(this.getClass(), SEARCH_EXECUTIONS_METRIC).inc(); return client.search(searchRequest, RequestOptions.DEFAULT); } catch (Exception e) { @@ -167,6 +231,32 @@ private SearchResponse executeLineageSearchQuery( } } + private BoolQueryBuilder getAggregationFilter( + Pair pair, RelationshipDirection direction) { + BoolQueryBuilder subFilter = QueryBuilders.boolQuery(); + TermQueryBuilder relationshipTypeTerm = + QueryBuilders.termQuery(RELATIONSHIP_TYPE, pair.getValue().getType()).caseInsensitive(true); + subFilter.must(relationshipTypeTerm); + + String sourceType; + String destinationType; + if (direction.equals(RelationshipDirection.OUTGOING)) { + sourceType = pair.getKey(); + destinationType = pair.getValue().getOpposingEntityType(); + } else { + sourceType = pair.getValue().getOpposingEntityType(); + destinationType = pair.getKey(); + } + + TermQueryBuilder sourceTypeTerm = + QueryBuilders.termQuery(SOURCE + ".entityType", sourceType).caseInsensitive(true); + subFilter.must(sourceTypeTerm); + TermQueryBuilder destinationTypeTerm = + QueryBuilders.termQuery(DESTINATION + ".entityType", destinationType).caseInsensitive(true); + subFilter.must(destinationTypeTerm); + return subFilter; + } + public SearchResponse getSearchResponse( @Nullable final List sourceTypes, @Nonnull final Filter sourceEntityFilter, @@ -412,16 +502,7 @@ private Stream processOneHopLineage( intermediateStream = Stream.concat(intermediateStream, ignoreAsHopUrns); } } - // We limit after adding all the relationships at the previous level so each hop is fully - // returned, - // but we only explore a limited number of entities per hop, sort to make the truncation - // consistent - if (lineageFlags.getEntitiesExploredPerHopLimit() != null) { - intermediateStream = - intermediateStream - .sorted(Comparator.comparing(Urn::toString)) - .limit(lineageFlags.getEntitiesExploredPerHopLimit()); - } + if (remainingHops > 0) { // If there are hops remaining, we expect to explore everything getting passed back to the // loop, barring a timeout @@ -537,11 +618,6 @@ private List getLineageRelationships( Collectors.toMap( Function.identity(), entityType -> lineageRegistry.getLineageRelationships(entityType, direction))); - - QueryBuilder finalQuery = - getLineageQuery(urnsPerEntityType, edgesPerEntityType, graphFilters, lineageFlags); - SearchResponse response = - executeLineageSearchQuery(finalQuery, 0, graphQueryConfiguration.getMaxResult()); Set entityUrnSet = new HashSet<>(entityUrns); // Get all valid edges given the set of urns to hop from Set> validEdges = @@ -550,16 +626,37 @@ private List getLineageRelationships( entry -> entry.getValue().stream().map(edgeInfo -> Pair.of(entry.getKey(), edgeInfo))) .collect(Collectors.toSet()); - return extractRelationships( - entityUrnSet, - response, - validEdges, - visitedEntities, - viaEntities, - numHops, - remainingHops, - existingPaths, - exploreMultiplePaths); + + QueryBuilder finalQuery = + getLineageQuery(urnsPerEntityType, edgesPerEntityType, graphFilters, lineageFlags); + SearchResponse response; + if (lineageFlags != null && lineageFlags.getEntitiesExploredPerHopLimit() != null) { + response = + executeGroupByLineageSearchQuery( + finalQuery, 0, lineageFlags.getEntitiesExploredPerHopLimit(), validEdges); + return extractRelationshipsGroupByQuery( + entityUrnSet, + response, + validEdges, + visitedEntities, + viaEntities, + numHops, + remainingHops, + existingPaths, + exploreMultiplePaths); + } else { + response = executeLineageSearchQuery(finalQuery, 0, graphQueryConfiguration.getMaxResult()); + return extractRelationships( + entityUrnSet, + response, + validEdges, + visitedEntities, + viaEntities, + numHops, + remainingHops, + existingPaths, + exploreMultiplePaths); + } } @VisibleForTesting @@ -756,158 +853,20 @@ private static List extractRelationships( log.debug("numHits: {}, numHops {}, remainingHops {}", hits.length, numHops, remainingHops); int index = -1; for (SearchHit hit : hits) { - index++; - final Map document = hit.getSourceAsMap(); - final Urn sourceUrn = - UrnUtils.getUrn(((Map) document.get(SOURCE)).get("urn").toString()); - final Urn destinationUrn = - UrnUtils.getUrn( - ((Map) document.get(DESTINATION)).get("urn").toString()); - final String type = document.get(RELATIONSHIP_TYPE).toString(); - if (sourceUrn.equals(destinationUrn)) { - log.debug("Skipping a self-edge of type {} on {}", type, sourceUrn); - continue; - } - final Number createdOnNumber = (Number) document.getOrDefault(CREATED_ON, null); - final Long createdOn = createdOnNumber != null ? createdOnNumber.longValue() : null; - final Number updatedOnNumber = (Number) document.getOrDefault(UPDATED_ON, null); - final Long updatedOn = updatedOnNumber != null ? updatedOnNumber.longValue() : null; - final String createdActorString = (String) document.getOrDefault(CREATED_ACTOR, null); - final Urn createdActor = - createdActorString == null ? null : UrnUtils.getUrn(createdActorString); - final String updatedActorString = (String) document.getOrDefault(UPDATED_ACTOR, null); - final Urn updatedActor = - updatedActorString == null ? null : UrnUtils.getUrn(updatedActorString); - final Map properties; - if (document.containsKey(PROPERTIES) && document.get(PROPERTIES) instanceof Map) { - properties = (Map) document.get(PROPERTIES); - } else { - properties = Collections.emptyMap(); - } - boolean isManual = properties.containsKey(SOURCE) && properties.get(SOURCE).equals("UI"); - Urn viaEntity = null; - String viaContent = (String) document.getOrDefault(EDGE_FIELD_VIA, null); - if (viaContent != null) { - try { - viaEntity = Urn.createFromString(viaContent); - } catch (Exception e) { - log.warn( - "Failed to parse urn from via entity {}, will swallow exception and continue...", - viaContent); - } - } - log.debug("{}: viaEntity {}", index, viaEntity); - - // Potential outgoing edge - if (entityUrns.contains(sourceUrn)) { - log.debug("{}: entity urns contains source urn {}", index, sourceUrn); - // Skip if already visited or if we're exploring multiple paths - // Skip if edge is not a valid outgoing edge - if ((exploreMultiplePaths || !visitedEntities.contains(destinationUrn)) - && validEdges.contains( - Pair.of( - sourceUrn.getEntityType(), - new EdgeInfo( - type, - RelationshipDirection.OUTGOING, - destinationUrn.getEntityType().toLowerCase())))) { - - if (visitedEntities.contains(destinationUrn)) { - log.debug("Found a second path to the same urn {}", destinationUrn); - } - // Append the edge to a set of unique graph paths. - if (addEdgeToPaths(existingPaths, sourceUrn, viaEntity, destinationUrn)) { - final LineageRelationship relationship = - createLineageRelationship( - type, - destinationUrn, - numHops, - existingPaths.getOrDefault(destinationUrn, new UrnArrayArray()), - // Fetch the paths to the next level entity. - createdOn, - createdActor, - updatedOn, - updatedActor, - isManual); - log.debug("Adding relationship {} to urn {}", relationship, destinationUrn); - lineageRelationshipMap.put(relationship.getEntity(), relationship); - if ((viaEntity != null) && (!viaEntities.contains(viaEntity))) { - UrnArrayArray viaPaths = getViaPaths(existingPaths, destinationUrn, viaEntity); - LineageRelationship viaRelationship = - createLineageRelationship( - type, - viaEntity, - numHops, - viaPaths, - createdOn, - createdActor, - updatedOn, - updatedActor, - isManual); - viaEntities.add(viaEntity); - lineageRelationshipMap.put(viaRelationship.getEntity(), viaRelationship); - log.debug("Adding via entity {} with paths {}", viaEntity, viaPaths); - } - } - visitedEntities.add(destinationUrn); - } - } - - // Potential incoming edge - if (entityUrns.contains(destinationUrn)) { - // Skip if already visited or if we're exploring multiple paths - // Skip if edge is not a valid outgoing edge - log.debug("entity urns contains destination urn {}", destinationUrn); - if ((exploreMultiplePaths || !visitedEntities.contains(sourceUrn)) - && validEdges.contains( - Pair.of( - destinationUrn.getEntityType(), - new EdgeInfo( - type, - RelationshipDirection.INCOMING, - sourceUrn.getEntityType().toLowerCase())))) { - if (visitedEntities.contains(sourceUrn)) { - log.debug("Found a second path to the same urn {}", sourceUrn); - } - visitedEntities.add(sourceUrn); - // Append the edge to a set of unique graph paths. - if (addEdgeToPaths(existingPaths, destinationUrn, viaEntity, sourceUrn)) { - log.debug("Adding incoming edge: {}, {}, {}", destinationUrn, viaEntity, sourceUrn); - final LineageRelationship relationship = - createLineageRelationship( - type, - sourceUrn, - numHops, - existingPaths.getOrDefault(sourceUrn, new UrnArrayArray()), - // Fetch the paths to the next level entity. - createdOn, - createdActor, - updatedOn, - updatedActor, - isManual); - log.debug("Adding relationship {} to urn {}", relationship, sourceUrn); - lineageRelationshipMap.put(relationship.getEntity(), relationship); - if ((viaEntity != null) && (!viaEntities.contains(viaEntity))) { - UrnArrayArray viaPaths = getViaPaths(existingPaths, sourceUrn, viaEntity); - viaEntities.add(viaEntity); - LineageRelationship viaRelationship = - createLineageRelationship( - type, - viaEntity, - numHops, - viaPaths, - createdOn, - createdActor, - updatedOn, - updatedActor, - isManual); - lineageRelationshipMap.put(viaRelationship.getEntity(), viaRelationship); - log.debug("Adding via relationship {} to urn {}", viaRelationship, viaEntity); - } - } - } - } + processSearchHit( + hit, + entityUrns, + index, + exploreMultiplePaths, + visitedEntities, + validEdges, + existingPaths, + numHops, + false, + lineageRelationshipMap, + viaEntities); } + List result = new ArrayList<>(lineageRelationshipMap.values()); log.debug("Number of lineage relationships in list: {}", result.size()); return result; @@ -919,6 +878,261 @@ private static List extractRelationships( } } + private static void processSearchHit( + SearchHit hit, + Set entityUrns, + int index, + boolean exploreMultiplePaths, + Set visitedEntities, + Set> validEdges, + Map existingPaths, + int numHops, + boolean truncatedChildren, + Map lineageRelationshipMap, + Set viaEntities) { + index++; + // Extract fields + final Map document = hit.getSourceAsMap(); + final Urn sourceUrn = + UrnUtils.getUrn(((Map) document.get(SOURCE)).get("urn").toString()); + final Urn destinationUrn = + UrnUtils.getUrn(((Map) document.get(DESTINATION)).get("urn").toString()); + final String type = document.get(RELATIONSHIP_TYPE).toString(); + if (sourceUrn.equals(destinationUrn)) { + log.debug("Skipping a self-edge of type {} on {}", type, sourceUrn); + return; + } + final Number createdOnNumber = (Number) document.getOrDefault(CREATED_ON, null); + final Long createdOn = createdOnNumber != null ? createdOnNumber.longValue() : null; + final Number updatedOnNumber = (Number) document.getOrDefault(UPDATED_ON, null); + final Long updatedOn = updatedOnNumber != null ? updatedOnNumber.longValue() : null; + final String createdActorString = (String) document.getOrDefault(CREATED_ACTOR, null); + final Urn createdActor = + createdActorString == null ? null : UrnUtils.getUrn(createdActorString); + final String updatedActorString = (String) document.getOrDefault(UPDATED_ACTOR, null); + final Urn updatedActor = + updatedActorString == null ? null : UrnUtils.getUrn(updatedActorString); + final Map properties; + if (document.containsKey(PROPERTIES) && document.get(PROPERTIES) instanceof Map) { + properties = (Map) document.get(PROPERTIES); + } else { + properties = Collections.emptyMap(); + } + boolean isManual = properties.containsKey(SOURCE) && properties.get(SOURCE).equals("UI"); + Urn viaEntity = null; + String viaContent = (String) document.getOrDefault(EDGE_FIELD_VIA, null); + if (viaContent != null) { + try { + viaEntity = Urn.createFromString(viaContent); + } catch (Exception e) { + log.warn( + "Failed to parse urn from via entity {}, will swallow exception and continue...", + viaContent); + } + } + log.debug("{}: viaEntity {}", index, viaEntity); + + // Potential outgoing edge + if (entityUrns.contains(sourceUrn)) { + processOutgoingEdge( + entityUrns, + sourceUrn, + index, + exploreMultiplePaths, + visitedEntities, + destinationUrn, + validEdges, + type, + existingPaths, + viaEntity, + numHops, + createdOn, + createdActor, + updatedOn, + updatedActor, + isManual, + truncatedChildren, + lineageRelationshipMap, + viaEntities); + } + + // Potential incoming edge + if (entityUrns.contains(destinationUrn)) { + processIncomingEdge( + entityUrns, + sourceUrn, + exploreMultiplePaths, + visitedEntities, + destinationUrn, + validEdges, + type, + existingPaths, + viaEntity, + numHops, + createdOn, + createdActor, + updatedOn, + updatedActor, + isManual, + truncatedChildren, + lineageRelationshipMap, + viaEntities); + } + } + + private static void processOutgoingEdge( + Set entityUrns, + Urn sourceUrn, + int index, + boolean exploreMultiplePaths, + Set visitedEntities, + Urn destinationUrn, + Set> validEdges, + String type, + Map existingPaths, + Urn viaEntity, + int numHops, + Long createdOn, + Urn createdActor, + Long updatedOn, + Urn updatedActor, + boolean isManual, + boolean truncatedChildren, + Map lineageRelationshipMap, + Set viaEntities) { + if (entityUrns.contains(sourceUrn)) { + log.debug("{}: entity urns contains source urn {}", index, sourceUrn); + // Skip if already visited or if we're exploring multiple paths + // Skip if edge is not a valid outgoing edge + if ((exploreMultiplePaths || !visitedEntities.contains(destinationUrn)) + && validEdges.contains( + Pair.of( + sourceUrn.getEntityType(), + new EdgeInfo( + type, + RelationshipDirection.OUTGOING, + destinationUrn.getEntityType().toLowerCase())))) { + + if (visitedEntities.contains(destinationUrn)) { + log.debug("Found a second path to the same urn {}", destinationUrn); + } + // Append the edge to a set of unique graph paths. + if (addEdgeToPaths(existingPaths, sourceUrn, viaEntity, destinationUrn)) { + final LineageRelationship relationship = + createLineageRelationship( + type, + destinationUrn, + numHops, + existingPaths.getOrDefault(destinationUrn, new UrnArrayArray()), + // Fetch the paths to the next level entity. + createdOn, + createdActor, + updatedOn, + updatedActor, + isManual, + truncatedChildren); + log.debug("Adding relationship {} to urn {}", relationship, destinationUrn); + lineageRelationshipMap.put(relationship.getEntity(), relationship); + if ((viaEntity != null) && (!viaEntities.contains(viaEntity))) { + UrnArrayArray viaPaths = getViaPaths(existingPaths, destinationUrn, viaEntity); + LineageRelationship viaRelationship = + createLineageRelationship( + type, + viaEntity, + numHops, + viaPaths, + createdOn, + createdActor, + updatedOn, + updatedActor, + isManual, + truncatedChildren); + viaEntities.add(viaEntity); + lineageRelationshipMap.put(viaRelationship.getEntity(), viaRelationship); + log.debug("Adding via entity {} with paths {}", viaEntity, viaPaths); + } + } + visitedEntities.add(destinationUrn); + } + } + } + + private static void processIncomingEdge( + Set entityUrns, + Urn sourceUrn, + boolean exploreMultiplePaths, + Set visitedEntities, + Urn destinationUrn, + Set> validEdges, + String type, + Map existingPaths, + Urn viaEntity, + int numHops, + Long createdOn, + Urn createdActor, + Long updatedOn, + Urn updatedActor, + boolean isManual, + boolean truncatedChildren, + Map lineageRelationshipMap, + Set viaEntities) { + if (entityUrns.contains(destinationUrn)) { + // Skip if already visited or if we're exploring multiple paths + // Skip if edge is not a valid outgoing edge + log.debug("entity urns contains destination urn {}", destinationUrn); + if ((exploreMultiplePaths || !visitedEntities.contains(sourceUrn)) + && validEdges.contains( + Pair.of( + destinationUrn.getEntityType(), + new EdgeInfo( + type, + RelationshipDirection.INCOMING, + sourceUrn.getEntityType().toLowerCase())))) { + if (visitedEntities.contains(sourceUrn)) { + log.debug("Found a second path to the same urn {}", sourceUrn); + } + visitedEntities.add(sourceUrn); + // Append the edge to a set of unique graph paths. + if (addEdgeToPaths(existingPaths, destinationUrn, viaEntity, sourceUrn)) { + log.debug("Adding incoming edge: {}, {}, {}", destinationUrn, viaEntity, sourceUrn); + final LineageRelationship relationship = + createLineageRelationship( + type, + sourceUrn, + numHops, + existingPaths.getOrDefault(sourceUrn, new UrnArrayArray()), + // Fetch the paths to the next level entity. + createdOn, + createdActor, + updatedOn, + updatedActor, + isManual, + truncatedChildren); + log.debug("Adding relationship {} to urn {}", relationship, sourceUrn); + lineageRelationshipMap.put(relationship.getEntity(), relationship); + if ((viaEntity != null) && (!viaEntities.contains(viaEntity))) { + UrnArrayArray viaPaths = getViaPaths(existingPaths, sourceUrn, viaEntity); + viaEntities.add(viaEntity); + LineageRelationship viaRelationship = + createLineageRelationship( + type, + viaEntity, + numHops, + viaPaths, + createdOn, + createdActor, + updatedOn, + updatedActor, + isManual, + truncatedChildren); + lineageRelationshipMap.put(viaRelationship.getEntity(), viaRelationship); + log.debug("Adding via relationship {} to urn {}", viaRelationship, viaEntity); + } + } + } + } + } + private static UrnArrayArray getViaPaths( Map existingPaths, Urn destinationUrn, Urn viaEntity) { UrnArrayArray destinationPaths = @@ -946,7 +1160,8 @@ private static LineageRelationship createLineageRelationship( @Nullable final Urn createdActor, @Nullable final Long updatedOn, @Nullable final Urn updatedActor, - final boolean isManual) { + final boolean isManual, + final boolean truncatedChildren) { final LineageRelationship relationship = new LineageRelationship() .setType(type) @@ -967,9 +1182,88 @@ private static LineageRelationship createLineageRelationship( relationship.setUpdatedActor(updatedActor); } relationship.setIsManual(isManual); + relationship.setTruncatedChildren(truncatedChildren); return relationship; } + @WithSpan + private static List extractRelationshipsGroupByQuery( + @Nonnull Set entityUrns, + @Nonnull SearchResponse searchResponse, + Set> validEdges, + Set visitedEntities, + Set viaEntities, + int numHops, + int remainingHops, + Map existingPaths, + boolean exploreMultiplePaths) { + try { + Map lineageRelationshipMap = new HashMap<>(); + ParsedFilter sourceFilterAgg = + searchResponse.getAggregations().get(FILTER_BY_SOURCE_RELATIONSHIP); + ParsedStringTerms sourceTermsAgg = sourceFilterAgg.getAggregations().get(GROUP_BY_SOURCE_AGG); + SearchHit[] hits = new SearchHit[0]; + List sourceBuckets = + (List) sourceTermsAgg.getBuckets(); + int index = -1; + for (ParsedStringTerms.ParsedBucket bucket : sourceBuckets) { + ParsedTopHits topHits = bucket.getAggregations().get(TOP_DOCUMENTS_AGG); + SearchHit[] topHitsArray = topHits.getHits().getHits(); + boolean truncatedChildren = topHits.getHits().getTotalHits().value > topHitsArray.length; + for (SearchHit hit : topHitsArray) { + processSearchHit( + hit, + entityUrns, + index, + exploreMultiplePaths, + visitedEntities, + validEdges, + existingPaths, + numHops, + truncatedChildren, + lineageRelationshipMap, + viaEntities); + } + } + + ParsedFilter destFilterAgg = + searchResponse.getAggregations().get(FILTER_BY_DESTINATION_RELATIONSHIP); + ParsedStringTerms destTermsAgg = + destFilterAgg.getAggregations().get(GROUP_BY_DESTINATION_AGG); + List destBuckets = + (List) destTermsAgg.getBuckets(); + for (ParsedStringTerms.ParsedBucket bucket : destBuckets) { + ParsedTopHits topHits = bucket.getAggregations().get(TOP_DOCUMENTS_AGG); + SearchHit[] topHitsArray = topHits.getHits().getHits(); + boolean truncatedChildren = topHits.getHits().getTotalHits().value > topHitsArray.length; + for (SearchHit hit : topHitsArray) { + processSearchHit( + hit, + entityUrns, + index, + exploreMultiplePaths, + visitedEntities, + validEdges, + existingPaths, + numHops, + truncatedChildren, + lineageRelationshipMap, + viaEntities); + } + } + log.debug("numHits: {}, numHops {}, remainingHops {}", hits.length, numHops, remainingHops); + + List result = new ArrayList<>(lineageRelationshipMap.values()); + log.debug("Number of lineage relationships in list: {}", result.size()); + return result; + } catch (Exception e) { + // This exception handler merely exists to log the exception at an appropriate point and + // rethrow + log.error("Caught exception", e); + throw e; + } + } + private static BoolQueryBuilder getOutGoingEdgeQuery( @Nonnull List urns, @Nonnull List outgoingEdges, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java index 37cc6ab6b1eeb..ada5069d0cabe 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java @@ -35,6 +35,7 @@ import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.util.Pair; import io.opentelemetry.extension.annotations.WithSpan; import java.io.IOException; import java.util.ArrayList; @@ -274,10 +275,10 @@ public void removeEdgesFromNode( } @Override - public void configure() { + public void reindexAll(Collection> properties) { log.info("Setting up elastic graph index"); try { - for (ReindexConfig config : buildReindexConfigs()) { + for (ReindexConfig config : buildReindexConfigs(properties)) { _indexBuilder.buildIndex(config); } } catch (IOException e) { @@ -286,7 +287,8 @@ public void configure() { } @Override - public List buildReindexConfigs() throws IOException { + public List buildReindexConfigs( + Collection> properties) throws IOException { return List.of( _indexBuilder.buildReindexState( _indexConvention.getIndexName(INDEX_NAME), @@ -294,17 +296,6 @@ public List buildReindexConfigs() throws IOException { Collections.emptyMap())); } - @Override - public List buildReindexConfigsWithAllStructProps( - Collection properties) throws IOException { - return buildReindexConfigs(); - } - - @Override - public void reindexAll() { - configure(); - } - @VisibleForTesting @Override public void clear() { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java index 70b30f27553c7..16c0804538dd7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java @@ -698,11 +698,6 @@ public void removeNodesMatchingLabel(@Nonnull String labelPattern) { runQuery(buildStatement(statement, params)).consume(); } - @Override - public void configure() { - // Do nothing - } - @Override public void clear() { removeNodesMatchingLabel(".*"); @@ -942,29 +937,27 @@ public RelatedEntitiesScrollResult scrollRelatedEntities( final String edgeCriteria = relationshipFilterToCriteria(relationshipFilter); final RelationshipDirection relationshipDirection = relationshipFilter.getDirection(); - String srcNodeLabel = ""; + + String matchTemplate = "MATCH (src %s)-[r%s %s]-(dest %s)%s"; + if (relationshipDirection == RelationshipDirection.INCOMING) { + matchTemplate = "MATCH (src %s)<-[r%s %s]-(dest %s)%s"; + } else if (relationshipDirection == RelationshipDirection.OUTGOING) { + matchTemplate = "MATCH (src %s)-[r%s %s]->(dest %s)%s"; + } + + String srcNodeLabel = StringUtils.EMPTY; // Create a URN from the String. Only proceed if srcCriteria is not null or empty - if (srcCriteria != null && !srcCriteria.isEmpty()) { + if (StringUtils.isNotEmpty(srcCriteria)) { final String urnValue = sourceEntityFilter.getOr().get(0).getAnd().get(0).getValue().toString(); try { final Urn urn = Urn.createFromString(urnValue); srcNodeLabel = urn.getEntityType(); + matchTemplate = matchTemplate.replace("(src ", "(src:%s "); } catch (URISyntaxException e) { log.error("Failed to parse URN: {} ", urnValue, e); } } - String matchTemplate = "MATCH (src:%s %s)-[r%s %s]-(dest %s)%s"; - if (relationshipDirection == RelationshipDirection.INCOMING) { - matchTemplate = "MATCH (src:%s %s)<-[r%s %s]-(dest %s)%s"; - } else if (relationshipDirection == RelationshipDirection.OUTGOING) { - matchTemplate = "MATCH (src:%s %s)-[r%s %s]->(dest %s)%s"; - } - - final String returnNodes = - String.format( - "RETURN dest, src, type(r)"); // Return both related entity and the relationship type. - final String returnCount = "RETURN count(*)"; // For getting the total results. String relationshipTypeFilter = ""; if (!relationshipTypes.isEmpty()) { @@ -974,18 +967,34 @@ public RelatedEntitiesScrollResult scrollRelatedEntities( String whereClause = computeEntityTypeWhereClause(sourceTypes, destinationTypes); // Build Statement strings - String baseStatementString = - String.format( - matchTemplate, - srcNodeLabel, - srcCriteria, - relationshipTypeFilter, - edgeCriteria, - destCriteria, - whereClause); + String baseStatementString; + if (StringUtils.isNotEmpty(srcNodeLabel)) { + baseStatementString = + String.format( + matchTemplate, + srcNodeLabel, + srcCriteria, + relationshipTypeFilter, + edgeCriteria, + destCriteria, + whereClause); + } else { + baseStatementString = + String.format( + matchTemplate, + srcCriteria, + relationshipTypeFilter, + edgeCriteria, + destCriteria, + whereClause); + } log.info(baseStatementString); + final String returnNodes = + "RETURN dest, src, type(r)"; // Return both related entity and the relationship type. + final String returnCount = "RETURN count(*)"; // For getting the total results. + final String resultStatementString = String.format("%s %s SKIP $offset LIMIT $count", baseStatementString, returnNodes); final String countStatementString = String.format("%s %s", baseStatementString, returnCount); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/MostPopularSource.java b/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/MostPopularSource.java index 399b0aa6e49a6..fea3fafdc845a 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/MostPopularSource.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/MostPopularSource.java @@ -146,7 +146,9 @@ private SearchRequest buildSearchRequest(@Nonnull OperationContext opContext) { // Find the entities with the most views AggregationBuilder aggregation = AggregationBuilders.terms(ENTITY_AGG_NAME) - .field(ESUtils.toKeywordField(DataHubUsageEventConstants.ENTITY_URN, false)) + .field( + ESUtils.toKeywordField( + DataHubUsageEventConstants.ENTITY_URN, false, opContext.getAspectRetriever())) .size(MAX_CONTENT * 2); source.aggregation(aggregation); source.size(0); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyEditedSource.java b/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyEditedSource.java index d75470127ded8..afdce0d714513 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyEditedSource.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyEditedSource.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.datahubusage.DataHubUsageEventConstants; import com.linkedin.metadata.datahubusage.DataHubUsageEventType; import com.linkedin.metadata.entity.EntityService; @@ -104,7 +105,8 @@ public List getRecommendations( @Nonnull RecommendationRequestContext requestContext, @Nullable Filter filter) { SearchRequest searchRequest = - buildSearchRequest(opContext.getSessionActorContext().getActorUrn()); + buildSearchRequest( + opContext.getSessionActorContext().getActorUrn(), opContext.getAspectRetriever()); try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "getRecentlyEdited").time()) { final SearchResponse searchResponse = _searchClient.search(searchRequest, RequestOptions.DEFAULT); @@ -128,7 +130,8 @@ public Set getSupportedEntityTypes() { return SUPPORTED_ENTITY_TYPES; } - private SearchRequest buildSearchRequest(@Nonnull Urn userUrn) { + private SearchRequest buildSearchRequest( + @Nonnull Urn userUrn, @Nullable AspectRetriever aspectRetriever) { // TODO: Proactively filter for entity types in the supported set. SearchRequest request = new SearchRequest(); SearchSourceBuilder source = new SearchSourceBuilder(); @@ -136,7 +139,7 @@ private SearchRequest buildSearchRequest(@Nonnull Urn userUrn) { // Filter for the entity edit events of the user requesting recommendation query.must( QueryBuilders.termQuery( - ESUtils.toKeywordField(DataHubUsageEventConstants.ACTOR_URN, false), + ESUtils.toKeywordField(DataHubUsageEventConstants.ACTOR_URN, false, aspectRetriever), userUrn.toString())); // Filter for the entity action events query.must( @@ -148,7 +151,9 @@ private SearchRequest buildSearchRequest(@Nonnull Urn userUrn) { String lastViewed = "last_viewed"; AggregationBuilder aggregation = AggregationBuilders.terms(ENTITY_AGG_NAME) - .field(ESUtils.toKeywordField(DataHubUsageEventConstants.ENTITY_URN, false)) + .field( + ESUtils.toKeywordField( + DataHubUsageEventConstants.ENTITY_URN, false, aspectRetriever)) .size(MAX_CONTENT) .order(BucketOrder.aggregation(lastViewed, false)) .subAggregation( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyViewedSource.java b/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyViewedSource.java index e9613495e8d22..f282470193ae5 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyViewedSource.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlyViewedSource.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.datahubusage.DataHubUsageEventConstants; import com.linkedin.metadata.datahubusage.DataHubUsageEventType; import com.linkedin.metadata.entity.EntityService; @@ -104,7 +105,8 @@ public List getRecommendations( @Nonnull RecommendationRequestContext requestContext, @Nullable Filter filter) { SearchRequest searchRequest = - buildSearchRequest(opContext.getSessionActorContext().getActorUrn()); + buildSearchRequest( + opContext.getSessionActorContext().getActorUrn(), opContext.getAspectRetriever()); try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "getRecentlyViewed").time()) { final SearchResponse searchResponse = _searchClient.search(searchRequest, RequestOptions.DEFAULT); @@ -128,7 +130,8 @@ public Set getSupportedEntityTypes() { return SUPPORTED_ENTITY_TYPES; } - private SearchRequest buildSearchRequest(@Nonnull Urn userUrn) { + private SearchRequest buildSearchRequest( + @Nonnull Urn userUrn, @Nullable AspectRetriever aspectRetriever) { // TODO: Proactively filter for entity types in the supported set. SearchRequest request = new SearchRequest(); SearchSourceBuilder source = new SearchSourceBuilder(); @@ -136,7 +139,7 @@ private SearchRequest buildSearchRequest(@Nonnull Urn userUrn) { // Filter for the entity view events of the user requesting recommendation query.must( QueryBuilders.termQuery( - ESUtils.toKeywordField(DataHubUsageEventConstants.ACTOR_URN, false), + ESUtils.toKeywordField(DataHubUsageEventConstants.ACTOR_URN, false, aspectRetriever), userUrn.toString())); query.must( QueryBuilders.termQuery( @@ -147,7 +150,9 @@ private SearchRequest buildSearchRequest(@Nonnull Urn userUrn) { String lastViewed = "last_viewed"; AggregationBuilder aggregation = AggregationBuilders.terms(ENTITY_AGG_NAME) - .field(ESUtils.toKeywordField(DataHubUsageEventConstants.ENTITY_URN, false)) + .field( + ESUtils.toKeywordField( + DataHubUsageEventConstants.ENTITY_URN, false, aspectRetriever)) .size(MAX_CONTENT) .order(BucketOrder.aggregation(lastViewed, false)) .subAggregation( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java index c06457768d725..95c8eb13beb93 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java @@ -745,6 +745,7 @@ private LineageSearchEntity buildLineageSearchEntity( entity.setDegrees(lineageRelationship.getDegrees()); } entity.setExplored(Boolean.TRUE.equals(lineageRelationship.isExplored())); + entity.setTruncatedChildren(Boolean.TRUE.equals(lineageRelationship.isTruncatedChildren())); entity.setIgnoredAsHop(Boolean.TRUE.equals(lineageRelationship.isIgnoredAsHop())); } return entity; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchServiceSearchRetriever.java b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchServiceSearchRetriever.java new file mode 100644 index 0000000000000..a5ef1c8fa58b1 --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchServiceSearchRetriever.java @@ -0,0 +1,51 @@ +package com.linkedin.metadata.search; + +import com.linkedin.metadata.entity.SearchRetriever; +import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.query.filter.SortOrder; +import io.datahubproject.metadata.context.OperationContext; +import java.util.List; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Builder +public class SearchServiceSearchRetriever implements SearchRetriever { + private static final SearchFlags RETRIEVER_SEARCH_FLAGS = + new SearchFlags() + .setFulltext(false) + .setMaxAggValues(20) + .setSkipCache(false) + .setSkipAggregates(true) + .setSkipHighlighting(true) + .setIncludeSoftDeleted(false) + .setIncludeRestricted(false); + + @Setter private OperationContext systemOperationContext; + private final SearchService searchService; + + @Override + public ScrollResult scroll( + @Nonnull List entities, + @Nullable Filter filters, + @Nullable String scrollId, + int count) { + SortCriterion urnSort = new SortCriterion(); + urnSort.setField("urn"); + urnSort.setOrder(SortOrder.ASCENDING); + return searchService.scrollAcrossEntities( + systemOperationContext.withSearchFlags(flags -> RETRIEVER_SEARCH_FLAGS), + entities, + "*", + filters, + urnSort, + scrollId, + null, + count); + } +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java index 19cd1f767f472..578c34611a75a 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java @@ -21,6 +21,7 @@ import com.linkedin.metadata.search.utils.SearchUtils; import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.util.Pair; import io.datahubproject.metadata.context.OperationContext; import java.io.IOException; import java.util.Collection; @@ -55,24 +56,14 @@ public class ElasticSearchService implements EntitySearchService, ElasticSearchI private final ESWriteDAO esWriteDAO; @Override - public void configure() { - indexBuilders.reindexAll(); + public void reindexAll(Collection> properties) { + indexBuilders.reindexAll(properties); } @Override - public List buildReindexConfigs() { - return indexBuilders.buildReindexConfigs(); - } - - @Override - public List buildReindexConfigsWithAllStructProps( - Collection properties) throws IOException { - return indexBuilders.buildReindexConfigsWithAllStructProps(properties); - } - - @Override - public void reindexAll() { - configure(); + public List buildReindexConfigs( + Collection> properties) throws IOException { + return indexBuilders.buildReindexConfigs(properties); } @Override diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java index cc6a0f3e3d6f9..2d04e99774050 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java @@ -81,6 +81,8 @@ public class ESIndexBuilder { @Getter private final boolean enableIndexMappingsReindex; + @Getter private final boolean enableStructuredPropertiesReindex; + @Getter private final ElasticSearchConfiguration elasticSearchConfiguration; @Getter private final GitVersion gitVersion; @@ -101,6 +103,7 @@ public ESIndexBuilder( Map> indexSettingOverrides, boolean enableIndexSettingsReindex, boolean enableIndexMappingsReindex, + boolean enableStructuredPropertiesReindex, ElasticSearchConfiguration elasticSearchConfiguration, GitVersion gitVersion) { this._searchClient = searchClient; @@ -112,6 +115,7 @@ public ESIndexBuilder( this.enableIndexSettingsReindex = enableIndexSettingsReindex; this.enableIndexMappingsReindex = enableIndexMappingsReindex; this.elasticSearchConfiguration = elasticSearchConfiguration; + this.enableStructuredPropertiesReindex = enableStructuredPropertiesReindex; this.gitVersion = gitVersion; RetryConfig config = @@ -143,6 +147,8 @@ public ReindexConfig buildReindexState( .name(indexName) .enableIndexSettingsReindex(enableIndexSettingsReindex) .enableIndexMappingsReindex(enableIndexMappingsReindex) + .enableStructuredPropertiesReindex( + enableStructuredPropertiesReindex && !copyStructuredPropertyMappings) .version(gitVersion.getVersion()); Map baseSettings = new HashMap<>(settings); @@ -293,7 +299,7 @@ public void buildIndex(ReindexConfig indexState) throws IOException { * @throws IOException communication issues with ES */ public void applyMappings(ReindexConfig indexState, boolean suppressError) throws IOException { - if (indexState.isPureMappingsAddition() || indexState.isPureStructuredProperty()) { + if (indexState.isPureMappingsAddition() || indexState.isPureStructuredPropertyAddition()) { log.info("Updating index {} mappings in place.", indexState.name()); PutMappingRequest request = new PutMappingRequest(indexState.name()).source(indexState.targetMappings()); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java index afc831b004ec3..eba4593c9042c 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java @@ -1,9 +1,11 @@ package com.linkedin.metadata.search.elasticsearch.indexbuilder; +import com.linkedin.common.urn.Urn; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.util.Pair; import java.io.IOException; import java.util.Collection; import java.util.List; @@ -26,8 +28,8 @@ public ESIndexBuilder getIndexBuilder() { } @Override - public void reindexAll() { - for (ReindexConfig config : buildReindexConfigs()) { + public void reindexAll(Collection> properties) { + for (ReindexConfig config : buildReindexConfigs(properties)) { try { indexBuilder.buildIndex(config); } catch (IOException e) { @@ -37,26 +39,8 @@ public void reindexAll() { } @Override - public List buildReindexConfigs() { - Map settings = settingsBuilder.getSettings(); - MappingsBuilder.setEntityRegistry(entityRegistry); - return entityRegistry.getEntitySpecs().values().stream() - .map( - entitySpec -> { - try { - Map mappings = MappingsBuilder.getMappings(entitySpec); - return indexBuilder.buildReindexState( - indexConvention.getIndexName(entitySpec), mappings, settings, true); - } catch (IOException e) { - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()); - } - - @Override - public List buildReindexConfigsWithAllStructProps( - Collection properties) { + public List buildReindexConfigs( + Collection> properties) { Map settings = settingsBuilder.getSettings(); MappingsBuilder.setEntityRegistry(entityRegistry); return entityRegistry.getEntitySpecs().values().stream() @@ -81,7 +65,7 @@ public List buildReindexConfigsWithAllStructProps( * @return index configurations impacted by the new property */ public List buildReindexConfigsWithNewStructProp( - StructuredPropertyDefinition property) { + Urn urn, StructuredPropertyDefinition property) { Map settings = settingsBuilder.getSettings(); MappingsBuilder.setEntityRegistry(entityRegistry); return entityRegistry.getEntitySpecs().values().stream() @@ -89,7 +73,7 @@ public List buildReindexConfigsWithNewStructProp( entitySpec -> { try { Map mappings = - MappingsBuilder.getMappings(entitySpec, List.of(property)); + MappingsBuilder.getMappings(entitySpec, List.of(Pair.of(urn, property))); return indexBuilder.buildReindexState( indexConvention.getIndexName(entitySpec), mappings, settings, true); } catch (IOException e) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java index f8d0f165bcddf..5dc28a8fd598d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java @@ -2,7 +2,7 @@ import static com.linkedin.metadata.Constants.ENTITY_TYPE_URN_PREFIX; import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD; -import static com.linkedin.metadata.models.StructuredPropertyUtils.sanitizeStructuredPropertyFQN; +import static com.linkedin.metadata.models.StructuredPropertyUtils.toElasticsearchFieldName; import static com.linkedin.metadata.models.annotation.SearchableAnnotation.OBJECT_FIELD_TYPES; import static com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder.*; @@ -17,11 +17,13 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.utils.ESUtils; import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.util.Pair; import java.net.URISyntaxException; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -52,6 +54,15 @@ public static Map getPartialNgramConfigWithOverrides( public static final String WORD_GRAMS_LENGTH_2 = "wordGrams2"; public static final String WORD_GRAMS_LENGTH_3 = "wordGrams3"; public static final String WORD_GRAMS_LENGTH_4 = "wordGrams4"; + public static final Set SUBFIELDS = + Set.of( + KEYWORD, + DELIMITED, + LENGTH, + NGRAM, + WORD_GRAMS_LENGTH_2, + WORD_GRAMS_LENGTH_3, + WORD_GRAMS_LENGTH_4); // Alias field mappings constants public static final String ALIAS = "alias"; @@ -72,7 +83,7 @@ private MappingsBuilder() {} */ public static Map getMappings( @Nonnull final EntitySpec entitySpec, - Collection structuredProperties) { + Collection> structuredProperties) { Map mappings = getMappings(entitySpec); String entityName = entitySpec.getEntityAnnotation().getName(); @@ -80,9 +91,11 @@ public static Map getMappings( getMappingsForStructuredProperty( structuredProperties.stream() .filter( - prop -> { + urnProp -> { try { - return prop.getEntityTypes() + return urnProp + .getSecond() + .getEntityTypes() .contains(Urn.createFromString(ENTITY_TYPE_URN_PREFIX + entityName)); } catch (URISyntaxException e) { return false; @@ -165,10 +178,11 @@ private static Map getMappingsForSystemCreated() { } public static Map getMappingsForStructuredProperty( - Collection properties) { + Collection> properties) { return properties.stream() .map( - property -> { + urnProperty -> { + StructuredPropertyDefinition property = urnProperty.getSecond(); Map mappingForField = new HashMap<>(); String valueType = property.getValueType().getId(); if (valueType.equalsIgnoreCase(LogicalValueType.STRING.name())) { @@ -183,7 +197,7 @@ public static Map getMappingsForStructuredProperty( mappingForField.put(TYPE, ESUtils.DOUBLE_FIELD_TYPE); } return Map.entry( - sanitizeStructuredPropertyFQN(property.getQualifiedName()), mappingForField); + toElasticsearchFieldName(urnProperty.getFirst(), property), mappingForField); }) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java index fbb7fcadba8bc..fd8db19ea6a7b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java @@ -11,6 +11,9 @@ import com.google.common.collect.MapDifference; import com.google.common.collect.Maps; import com.linkedin.metadata.search.utils.ESUtils; +import com.linkedin.util.Pair; +import java.util.AbstractMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -61,6 +64,7 @@ public class ReindexConfig { private final Map targetMappings; private final boolean enableIndexMappingsReindex; private final boolean enableIndexSettingsReindex; + private final boolean enableStructuredPropertiesReindex; private final String version; /* Calculated */ @@ -70,7 +74,8 @@ public class ReindexConfig { private final boolean isPureMappingsAddition; private final boolean isSettingsReindex; private final boolean hasNewStructuredProperty; - private final boolean isPureStructuredProperty; + private final boolean isPureStructuredPropertyAddition; + private final boolean hasRemovedStructuredProperty; public static ReindexConfigBuilder builder() { return new CalculatedBuilder(); @@ -102,7 +107,11 @@ private ReindexConfigBuilder hasNewStructuredProperty(boolean ignored) { return this; } - private ReindexConfigBuilder isPureStructuredProperty(boolean ignored) { + private ReindexConfigBuilder isPureStructuredPropertyAddition(boolean ignored) { + return this; + } + + private ReindexConfigBuilder hasRemovedStructuredProperty(boolean ignored) { return this; } @@ -156,7 +165,7 @@ public ReindexConfig build() { super.requiresApplyMappings = !mappingsDiff.entriesDiffering().isEmpty() || !mappingsDiff.entriesOnlyOnRight().isEmpty(); - super.isPureStructuredProperty = + super.isPureStructuredPropertyAddition = mappingsDiff .entriesDiffering() .keySet() @@ -169,6 +178,22 @@ public ReindexConfig build() { super.requiresApplyMappings && mappingsDiff.entriesDiffering().isEmpty() && !mappingsDiff.entriesOnlyOnRight().isEmpty(); + super.hasNewStructuredProperty = + (mappingsDiff.entriesDiffering().containsKey(STRUCTURED_PROPERTY_MAPPING_FIELD) + || mappingsDiff + .entriesOnlyOnRight() + .containsKey(STRUCTURED_PROPERTY_MAPPING_FIELD)) + && structuredPropertiesDiffCount(super.currentMappings, super.targetMappings) + .getSecond() + > 0; + super.hasRemovedStructuredProperty = + (mappingsDiff.entriesDiffering().containsKey(STRUCTURED_PROPERTY_MAPPING_FIELD) + || mappingsDiff + .entriesOnlyOnLeft() + .containsKey(STRUCTURED_PROPERTY_MAPPING_FIELD)) + && structuredPropertiesDiffCount(super.currentMappings, super.targetMappings) + .getFirst() + > 0; if (super.requiresApplyMappings && super.isPureMappingsAddition) { log.info( @@ -181,19 +206,6 @@ public ReindexConfig build() { super.name, mappingsDiff.entriesDiffering()); } - super.hasNewStructuredProperty = - (mappingsDiff.entriesDiffering().containsKey(STRUCTURED_PROPERTY_MAPPING_FIELD) - || mappingsDiff - .entriesOnlyOnRight() - .containsKey(STRUCTURED_PROPERTY_MAPPING_FIELD)) - && getOrDefault( - super.currentMappings, - List.of("properties", STRUCTURED_PROPERTY_MAPPING_FIELD, "properties")) - .size() - < getOrDefault( - super.targetMappings, - List.of("properties", STRUCTURED_PROPERTY_MAPPING_FIELD, "properties")) - .size(); /* Consider analysis and settings changes */ super.requiresApplySettings = !isSettingsEqual() || !isAnalysisEqual(); @@ -208,7 +220,26 @@ < getOrDefault( "Index: {} - There's diff between new mappings, however reindexing is DISABLED.", super.name); } + } else if (super.hasRemovedStructuredProperty) { + if (super.enableIndexMappingsReindex + && super.enableIndexMappingsReindex + && super.enableStructuredPropertiesReindex) { + super.requiresApplyMappings = true; + super.requiresReindex = true; + } else { + if (!super.enableIndexMappingsReindex) { + log.warn( + "Index: {} - There's diff between new mappings, however reindexing is DISABLED.", + super.name); + } + if (!super.enableIndexMappingsReindex) { + log.warn( + "Index: {} - There's a removed Structured Property, however Structured Property reindexing is DISABLED.", + super.name); + } + } } + if (super.isSettingsReindex) { try { if (!isAnalysisEqual()) { @@ -255,6 +286,46 @@ private static TreeMap getOrDefault( } } + /** + * Return counts for removed and added structured properties based on the difference between the + * existing mapping configuration and the target configuration + * + * @return count of structured properties to be removed and added to the index mapping + */ + private static Pair structuredPropertiesDiffCount( + Map current, Map target) { + Set currentStructuredProperties = new HashSet<>(); + Set targetStructuredProperties = new HashSet<>(); + + // add non-versioned property ids + currentStructuredProperties.addAll( + getOrDefault( + current, List.of("properties", STRUCTURED_PROPERTY_MAPPING_FIELD, "properties")) + .keySet() + .stream() + .filter(k -> !STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD.equals(k)) + .collect(Collectors.toSet())); + targetStructuredProperties.addAll( + getOrDefault( + target, List.of("properties", STRUCTURED_PROPERTY_MAPPING_FIELD, "properties")) + .keySet() + .stream() + .filter(k -> !STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD.equals(k)) + .collect(Collectors.toSet())); + + // Extract versioned/typed property ids + currentStructuredProperties.addAll(getVersionedStructuredPropertyIds(current)); + targetStructuredProperties.addAll(getVersionedStructuredPropertyIds(target)); + + return Pair.of( + currentStructuredProperties.stream() + .filter(p -> !targetStructuredProperties.contains(p)) + .count(), + targetStructuredProperties.stream() + .filter(p -> !currentStructuredProperties.contains(p)) + .count()); + } + private boolean isAnalysisEqual() { if (super.targetSettings == null || !super.targetSettings.containsKey("index")) { return true; @@ -327,7 +398,7 @@ private static MapDifference calculateMapDifference( .collect(Collectors.toSet()); if (!targetObjectFields.isEmpty()) { - log.info("Object fields filtered from comparison: {}", targetObjectFields); + log.debug("Object fields filtered from comparison: {}", targetObjectFields); Map filteredCurrentMappings = removeKeys(currentMappings, targetObjectFields); Map filteredTargetMappings = removeKeys(targetMappings, targetObjectFields); @@ -336,6 +407,29 @@ private static MapDifference calculateMapDifference( return Maps.difference(currentMappings, targetMappings); } + + /** + * Given a mapping return a unique string for each version/typed structured property + * + * @param mappings Elastic mappings + * @return set of unique ids for each versioned/typed structured property + */ + private static Set getVersionedStructuredPropertyIds(Map mappings) { + Map versionedMappings = + getOrDefault( + mappings, + List.of( + "properties", + STRUCTURED_PROPERTY_MAPPING_FIELD, + "properties", + STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD, + "properties")); + + return flattenStructuredPropertyPath( + Map.entry(STRUCTURED_PROPERTY_MAPPING_VERSIONED_FIELD, versionedMappings), 0) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + } } private static Map removeKeys( @@ -373,4 +467,22 @@ private static boolean equalsGroup(Map newSettings, Settings old } return true; } + + /** + * Return a map with dot delimited path as keys + * + * @param entry for root map + * @return dot delimited key path map + */ + private static Stream> flattenStructuredPropertyPath( + Map.Entry entry, int depth) { + if (entry.getValue() instanceof Map && depth < 5) { + Map nested = (Map) entry.getValue(); + + return nested.entrySet().stream() + .map(e -> new AbstractMap.SimpleEntry(entry.getKey() + "." + e.getKey(), e.getValue())) + .flatMap(e -> flattenStructuredPropertyPath(e, depth + 1)); + } + return Stream.of(entry); + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java index 97ec9f2192e52..b55418d12c7c2 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java @@ -607,11 +607,7 @@ private QueryBuilder buildQueryStringV2( EntitySpec entitySpec = opContext.getEntityRegistry().getEntitySpec(entityName); QueryBuilder query = - SearchRequestHandler.getBuilder( - entitySpec, - searchConfiguration, - customSearchConfiguration, - opContext.getRetrieverContext().get().getAspectRetriever()) + SearchRequestHandler.getBuilder(entitySpec, searchConfiguration, customSearchConfiguration) .getQuery( finalOpContext, input, @@ -647,11 +643,7 @@ private QueryBuilder buildQueryStringBrowseAcrossEntities( final BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); QueryBuilder query = - SearchRequestHandler.getBuilder( - entitySpecs, - searchConfiguration, - customSearchConfiguration, - opContext.getRetrieverContext().get().getAspectRetriever()) + SearchRequestHandler.getBuilder(entitySpecs, searchConfiguration, customSearchConfiguration) .getQuery( finalOpContext, input, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java index d8c5c3317a2ec..b537a39634027 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java @@ -115,10 +115,7 @@ private SearchResult executeAndExtract( return transformIndexIntoEntityName( opContext.getSearchContext().getIndexConvention(), SearchRequestHandler.getBuilder( - entitySpec, - searchConfiguration, - customSearchConfiguration, - opContext.getRetrieverContext().get().getAspectRetriever()) + entitySpec, searchConfiguration, customSearchConfiguration) .extractResult(opContext, searchResponse, filter, from, size)); } catch (Exception e) { log.error("Search query failed", e); @@ -215,10 +212,7 @@ private ScrollResult executeAndExtract( return transformIndexIntoEntityName( opContext.getSearchContext().getIndexConvention(), SearchRequestHandler.getBuilder( - entitySpecs, - searchConfiguration, - customSearchConfiguration, - opContext.getRetrieverContext().get().getAspectRetriever()) + entitySpecs, searchConfiguration, customSearchConfiguration) .extractScrollResult( opContext, searchResponse, filter, keepAlive, size, supportsPointInTime())); } catch (Exception e) { @@ -261,11 +255,7 @@ public SearchResult search( Filter transformedFilters = transformFilterForEntities(postFilters, indexConvention); // Step 1: construct the query final SearchRequest searchRequest = - SearchRequestHandler.getBuilder( - entitySpecs, - searchConfiguration, - customSearchConfiguration, - opContext.getRetrieverContext().get().getAspectRetriever()) + SearchRequestHandler.getBuilder(entitySpecs, searchConfiguration, customSearchConfiguration) .getSearchRequest( opContext, finalInput, transformedFilters, sortCriterion, from, size, facets); searchRequest.indices( @@ -298,11 +288,7 @@ public SearchResult filter( EntitySpec entitySpec = opContext.getEntityRegistry().getEntitySpec(entityName); Filter transformedFilters = transformFilterForEntities(filters, indexConvention); final SearchRequest searchRequest = - SearchRequestHandler.getBuilder( - entitySpec, - searchConfiguration, - customSearchConfiguration, - opContext.getRetrieverContext().get().getAspectRetriever()) + SearchRequestHandler.getBuilder(entitySpec, searchConfiguration, customSearchConfiguration) .getFilterRequest(opContext, transformedFilters, sortCriterion, from, size); searchRequest.indices(indexConvention.getIndexName(entitySpec)); @@ -335,10 +321,7 @@ public AutoCompleteResult autoComplete( EntitySpec entitySpec = opContext.getEntityRegistry().getEntitySpec(entityName); IndexConvention indexConvention = opContext.getSearchContext().getIndexConvention(); AutocompleteRequestHandler builder = - AutocompleteRequestHandler.getBuilder( - entitySpec, - customSearchConfiguration, - opContext.getRetrieverContext().get().getAspectRetriever()); + AutocompleteRequestHandler.getBuilder(entitySpec, customSearchConfiguration); SearchRequest req = builder.getSearchRequest( opContext, @@ -383,11 +366,7 @@ public Map aggregateByValue( } IndexConvention indexConvention = opContext.getSearchContext().getIndexConvention(); final SearchRequest searchRequest = - SearchRequestHandler.getBuilder( - entitySpecs, - searchConfiguration, - customSearchConfiguration, - opContext.getRetrieverContext().get().getAspectRetriever()) + SearchRequestHandler.getBuilder(entitySpecs, searchConfiguration, customSearchConfiguration) .getAggregationRequest( opContext, field, @@ -502,10 +481,7 @@ private SearchRequest getScrollRequest( } return SearchRequestHandler.getBuilder( - entitySpecs, - searchConfiguration, - customSearchConfiguration, - opContext.getRetrieverContext().get().getAspectRetriever()) + entitySpecs, searchConfiguration, customSearchConfiguration) .getSearchRequest( opContext, finalInput, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java index 971cd7298639d..fa2eef964e006 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java @@ -1,13 +1,14 @@ package com.linkedin.metadata.search.elasticsearch.query.request; import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.search.utils.ESUtils.toFacetField; +import static com.linkedin.metadata.models.StructuredPropertyUtils.toStructuredPropertyFacetName; +import static com.linkedin.metadata.search.utils.ESUtils.toParentField; import static com.linkedin.metadata.utils.SearchUtil.*; import com.linkedin.data.template.LongMap; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.models.EntitySpec; -import com.linkedin.metadata.models.StructuredPropertyUtils; import com.linkedin.metadata.models.annotation.SearchableAnnotation; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; @@ -136,14 +137,7 @@ private AggregationBuilder facetToAggregationBuilder( configs.getMaxTermBucketSize()); for (int i = facets.size() - 1; i >= 0; i--) { String facet = facets.get(i); - if (facet.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX)) { - String structPropFqn = facet.substring(STRUCTURED_PROPERTY_MAPPING_FIELD.length() + 1); - StructuredPropertyUtils.validateStructuredPropertyFQN( - Set.of(structPropFqn), opContext.getRetrieverContext().get().getAspectRetriever()); - facet = - STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX - + StructuredPropertyUtils.sanitizeStructuredPropertyFQN(structPropFqn); - } + AggregationBuilder aggBuilder; if (facet.contains(AGGREGATION_SPECIAL_TYPE_DELIMITER)) { List specialTypeFields = List.of(facet.split(AGGREGATION_SPECIAL_TYPE_DELIMITER)); @@ -152,9 +146,11 @@ private AggregationBuilder facetToAggregationBuilder( aggBuilder = INDEX_VIRTUAL_FIELD.equalsIgnoreCase(specialTypeFields.get(1)) ? AggregationBuilders.missing(inputFacet) - .field(getAggregationField(ES_INDEX_FIELD)) + .field(getAggregationField(ES_INDEX_FIELD, opContext.getAspectRetriever())) : AggregationBuilders.missing(inputFacet) - .field(getAggregationField(specialTypeFields.get(1))); + .field( + getAggregationField( + specialTypeFields.get(1), opContext.getAspectRetriever())); break; default: throw new UnsupportedOperationException( @@ -164,11 +160,11 @@ private AggregationBuilder facetToAggregationBuilder( aggBuilder = facet.equalsIgnoreCase(INDEX_VIRTUAL_FIELD) ? AggregationBuilders.terms(inputFacet) - .field(getAggregationField(ES_INDEX_FIELD)) + .field(getAggregationField(ES_INDEX_FIELD, opContext.getAspectRetriever())) .size(maxTermBuckets) .minDocCount(0) : AggregationBuilders.terms(inputFacet) - .field(getAggregationField(facet)) + .field(getAggregationField(facet, opContext.getAspectRetriever())) .size(maxTermBuckets); } if (lastAggBuilder != null) { @@ -180,13 +176,16 @@ private AggregationBuilder facetToAggregationBuilder( return lastAggBuilder; } - private String getAggregationField(final String facet) { + private String getAggregationField( + final String facet, @Nullable AspectRetriever aspectRetriever) { if (facet.startsWith("has")) { // Boolean hasX field, not a keyword field. Return the name of the original facet. return facet; } - // Otherwise assume that this field is of keyword type. - return ESUtils.toKeywordField(facet, false); + // intercept structured property if it exists + return toStructuredPropertyFacetName(facet, aspectRetriever) + // Otherwise assume that this field is of keyword type. + .orElse(ESUtils.toKeywordField(facet, false, aspectRetriever)); } List getDefaultFacetFieldsFromAnnotation(final SearchableAnnotation annotation) { @@ -229,10 +228,12 @@ private String computeDisplayName(String name) { } List extractAggregationMetadata( - @Nonnull SearchResponse searchResponse, @Nullable Filter filter) { + @Nonnull SearchResponse searchResponse, + @Nullable Filter filter, + @Nullable AspectRetriever aspectRetriever) { final List aggregationMetadataList = new ArrayList<>(); if (searchResponse.getAggregations() == null) { - return addFiltersToAggregationMetadata(aggregationMetadataList, filter); + return addFiltersToAggregationMetadata(aggregationMetadataList, filter, aspectRetriever); } for (Map.Entry entry : searchResponse.getAggregations().getAsMap().entrySet()) { @@ -243,7 +244,7 @@ List extractAggregationMetadata( processMissingAggregations(entry, aggregationMetadataList); } } - return addFiltersToAggregationMetadata(aggregationMetadataList, filter); + return addFiltersToAggregationMetadata(aggregationMetadataList, filter, aspectRetriever); } private void processTermAggregations( @@ -343,38 +344,45 @@ private static Map extractTermAggregations( /** Injects the missing conjunctive filters into the aggregations list. */ public List addFiltersToAggregationMetadata( - @Nonnull final List originalMetadata, @Nullable final Filter filter) { + @Nonnull final List originalMetadata, + @Nullable final Filter filter, + @Nullable AspectRetriever aspectRetriever) { if (filter == null) { return originalMetadata; } if (filter.getOr() != null) { - addOrFiltersToAggregationMetadata(filter.getOr(), originalMetadata); + addOrFiltersToAggregationMetadata(filter.getOr(), originalMetadata, aspectRetriever); } else if (filter.getCriteria() != null) { - addCriteriaFiltersToAggregationMetadata(filter.getCriteria(), originalMetadata); + addCriteriaFiltersToAggregationMetadata( + filter.getCriteria(), originalMetadata, aspectRetriever); } return originalMetadata; } void addOrFiltersToAggregationMetadata( @Nonnull final ConjunctiveCriterionArray or, - @Nonnull final List originalMetadata) { + @Nonnull final List originalMetadata, + @Nullable AspectRetriever aspectRetriever) { for (ConjunctiveCriterion conjunction : or) { // For each item in the conjunction, inject an empty aggregation if necessary - addCriteriaFiltersToAggregationMetadata(conjunction.getAnd(), originalMetadata); + addCriteriaFiltersToAggregationMetadata( + conjunction.getAnd(), originalMetadata, aspectRetriever); } } private void addCriteriaFiltersToAggregationMetadata( @Nonnull final CriterionArray criteria, - @Nonnull final List originalMetadata) { + @Nonnull final List originalMetadata, + @Nullable AspectRetriever aspectRetriever) { for (Criterion criterion : criteria) { - addCriterionFiltersToAggregationMetadata(criterion, originalMetadata); + addCriterionFiltersToAggregationMetadata(criterion, originalMetadata, aspectRetriever); } } private void addCriterionFiltersToAggregationMetadata( @Nonnull final Criterion criterion, - @Nonnull final List aggregationMetadata) { + @Nonnull final List aggregationMetadata, + @Nullable AspectRetriever aspectRetriever) { // We should never see duplicate aggregation for the same field in aggregation metadata list. final Map aggregationMetadataMap = @@ -382,7 +390,7 @@ private void addCriterionFiltersToAggregationMetadata( .collect(Collectors.toMap(AggregationMetadata::getName, agg -> agg)); // Map a filter criterion to a facet field (e.g. domains.keyword -> domains) - final String finalFacetField = toFacetField(criterion.getField()); + final String finalFacetField = toParentField(criterion.getField(), aspectRetriever); if (finalFacetField == null) { log.warn( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java index 3768fbdb01884..8ee9587ca2ae4 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java @@ -8,7 +8,6 @@ import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.StringArray; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.config.search.custom.AutocompleteConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.config.search.custom.QueryConfiguration; @@ -37,10 +36,7 @@ import lombok.extern.slf4j.Slf4j; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; -import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.MultiMatchQueryBuilder; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.*; import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; @@ -55,16 +51,13 @@ public class AutocompleteRequestHandler { private static final Map AUTOCOMPLETE_QUERY_BUILDER_BY_ENTITY_NAME = new ConcurrentHashMap<>(); - private final AspectRetriever aspectRetriever; - private final CustomizedQueryHandler customizedQueryHandler; private final EntitySpec entitySpec; public AutocompleteRequestHandler( @Nonnull EntitySpec entitySpec, - @Nullable CustomSearchConfiguration customSearchConfiguration, - @Nonnull AspectRetriever aspectRetriever) { + @Nullable CustomSearchConfiguration customSearchConfiguration) { this.entitySpec = entitySpec; List fieldSpecs = entitySpec.getSearchableFieldSpecs(); this.customizedQueryHandler = CustomizedQueryHandler.builder(customSearchConfiguration).build(); @@ -90,17 +83,13 @@ public AutocompleteRequestHandler( set1.addAll(set2); return set1; })); - this.aspectRetriever = aspectRetriever; } public static AutocompleteRequestHandler getBuilder( @Nonnull EntitySpec entitySpec, - @Nullable CustomSearchConfiguration customSearchConfiguration, - @Nonnull AspectRetriever aspectRetriever) { + @Nullable CustomSearchConfiguration customSearchConfiguration) { return AUTOCOMPLETE_QUERY_BUILDER_BY_ENTITY_NAME.computeIfAbsent( - entitySpec, - k -> - new AutocompleteRequestHandler(entitySpec, customSearchConfiguration, aspectRetriever)); + entitySpec, k -> new AutocompleteRequestHandler(entitySpec, customSearchConfiguration)); } public SearchRequest getSearchRequest( @@ -123,7 +112,8 @@ public SearchRequest getSearchRequest( // Initial query with input filters BoolQueryBuilder filterQuery = - ESUtils.buildFilterQuery(filter, false, searchableFieldTypes, aspectRetriever); + ESUtils.buildFilterQuery( + filter, false, searchableFieldTypes, opContext.getAspectRetriever()); baseQuery.filter(filterQuery); // Add autocomplete query @@ -219,10 +209,9 @@ private static BoolQueryBuilder defaultQuery( autocompleteQueryBuilder.field(fieldName + ".ngram._3gram"); autocompleteQueryBuilder.field(fieldName + ".ngram._4gram"); } - + autocompleteQueryBuilder.field(fieldName + ".delimited"); finalQuery.should(QueryBuilders.matchPhrasePrefixQuery(fieldName + ".delimited", query)); }); - finalQuery.should(autocompleteQueryBuilder); return finalQuery; } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java index 9262d17ccde92..6a1f24a92e4a3 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java @@ -7,6 +7,7 @@ import static com.linkedin.metadata.search.elasticsearch.query.request.CustomizedQueryHandler.unquote; import com.google.common.annotations.VisibleForTesting; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.config.search.ExactMatchConfiguration; import com.linkedin.metadata.config.search.PartialConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; @@ -109,7 +110,11 @@ private QueryBuilder buildInternalQuery( getSimpleQuery(opContext.getEntityRegistry(), customQueryConfig, entitySpecs, sanitizedQuery) .ifPresent(finalQuery::should); getPrefixAndExactMatchQuery( - opContext.getEntityRegistry(), customQueryConfig, entitySpecs, sanitizedQuery) + opContext.getEntityRegistry(), + customQueryConfig, + entitySpecs, + sanitizedQuery, + opContext.getAspectRetriever()) .ifPresent(finalQuery::should); } else { final String withoutQueryPrefix = @@ -121,7 +126,11 @@ private QueryBuilder buildInternalQuery( .ifPresent(finalQuery::should); if (exactMatchConfiguration.isEnableStructured()) { getPrefixAndExactMatchQuery( - opContext.getEntityRegistry(), customQueryConfig, entitySpecs, withoutQueryPrefix) + opContext.getEntityRegistry(), + customQueryConfig, + entitySpecs, + withoutQueryPrefix, + opContext.getAspectRetriever()) .ifPresent(finalQuery::should); } } @@ -369,7 +378,8 @@ private Optional getPrefixAndExactMatchQuery( @Nonnull EntityRegistry entityRegistry, @Nullable QueryConfiguration customQueryConfig, @Nonnull List entitySpecs, - String query) { + String query, + @Nullable AspectRetriever aspectRetriever) { final boolean isPrefixQuery = customQueryConfig == null @@ -408,7 +418,8 @@ private Optional getPrefixAndExactMatchQuery( if (caseSensitivityEnabled) { finalQuery.should( QueryBuilders.termQuery( - ESUtils.toKeywordField(searchFieldConfig.fieldName(), false), + ESUtils.toKeywordField( + searchFieldConfig.fieldName(), false, aspectRetriever), unquotedQuery) .caseInsensitive(false) .boost( @@ -419,7 +430,8 @@ private Optional getPrefixAndExactMatchQuery( // Exact match case-insensitive finalQuery.should( QueryBuilders.termQuery( - ESUtils.toKeywordField(searchFieldConfig.fieldName(), false), + ESUtils.toKeywordField( + searchFieldConfig.fieldName(), false, aspectRetriever), unquotedQuery) .caseInsensitive(true) .boost( @@ -432,7 +444,8 @@ private Optional getPrefixAndExactMatchQuery( if (searchFieldConfig.isWordGramSubfield() && isPrefixQuery) { finalQuery.should( QueryBuilders.matchPhraseQuery( - ESUtils.toKeywordField(searchFieldConfig.fieldName(), false), + ESUtils.toKeywordField( + searchFieldConfig.fieldName(), false, aspectRetriever), unquotedQuery) .boost( searchFieldConfig.boost() @@ -455,7 +468,7 @@ private Optional getStructuredQuery( if (customQueryConfig != null) { executeStructuredQuery = customQueryConfig.isStructuredQuery(); } else { - executeStructuredQuery = !(isQuoted(sanitizedQuery) && exactMatchConfiguration.isExclusive()); + executeStructuredQuery = true; } if (executeStructuredQuery) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java index dfb33897bcf4a..66ad1e3be363f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java @@ -8,7 +8,6 @@ import com.google.common.collect.ImmutableMap; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.DoubleMap; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.models.EntitySpec; @@ -117,8 +116,7 @@ private SearchRequestHandler( public static SearchRequestHandler getBuilder( @Nonnull EntitySpec entitySpec, @Nonnull SearchConfiguration configs, - @Nullable CustomSearchConfiguration customSearchConfiguration, - @Nonnull AspectRetriever aspectRetriever) { + @Nullable CustomSearchConfiguration customSearchConfiguration) { return REQUEST_HANDLER_BY_ENTITY_NAME.computeIfAbsent( ImmutableList.of(entitySpec), k -> new SearchRequestHandler(entitySpec, configs, customSearchConfiguration)); @@ -127,8 +125,7 @@ public static SearchRequestHandler getBuilder( public static SearchRequestHandler getBuilder( @Nonnull List entitySpecs, @Nonnull SearchConfiguration configs, - @Nullable CustomSearchConfiguration customSearchConfiguration, - @Nonnull AspectRetriever aspectRetriever) { + @Nullable CustomSearchConfiguration customSearchConfiguration) { return REQUEST_HANDLER_BY_ENTITY_NAME.computeIfAbsent( ImmutableList.copyOf(entitySpecs), k -> new SearchRequestHandler(entitySpecs, configs, customSearchConfiguration)); @@ -167,10 +164,7 @@ public static BoolQueryBuilder getFilterQuery( Map> searchableFieldTypes) { BoolQueryBuilder filterQuery = ESUtils.buildFilterQuery( - filter, - false, - searchableFieldTypes, - opContext.getRetrieverContext().get().getAspectRetriever()); + filter, false, searchableFieldTypes, opContext.getAspectRetriever()); return applyDefaultSearchFilters(opContext, filter, filterQuery); } @@ -337,7 +331,9 @@ public SearchRequest getAggregationRequest( searchSourceBuilder.query(filterQuery); searchSourceBuilder.size(0); searchSourceBuilder.aggregation( - AggregationBuilders.terms(field).field(ESUtils.toKeywordField(field, false)).size(limit)); + AggregationBuilders.terms(field) + .field(ESUtils.toKeywordField(field, false, opContext.getAspectRetriever())) + .size(limit)); searchRequest.source(searchSourceBuilder); return searchRequest; @@ -528,7 +524,8 @@ private SearchResultMetadata extractSearchResultMetadata( if (Boolean.FALSE.equals(searchFlags.isSkipAggregates())) { final List aggregationMetadataList = - aggregationQueryBuilder.extractAggregationMetadata(searchResponse, filter); + aggregationQueryBuilder.extractAggregationMetadata( + searchResponse, filter, opContext.getAspectRetriever()); searchResultMetadata.setAggregations(new AggregationMetadataArray(aggregationMetadataList)); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java index a655f90597e20..dd36f0a9456a7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java @@ -1,7 +1,7 @@ package com.linkedin.metadata.search.transformer; import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.models.StructuredPropertyUtils.sanitizeStructuredPropertyFQN; +import static com.linkedin.metadata.models.StructuredPropertyUtils.toElasticsearchFieldName; import static com.linkedin.metadata.models.annotation.SearchableAnnotation.OBJECT_FIELD_TYPES; import static com.linkedin.metadata.search.elasticsearch.indexbuilder.MappingsBuilder.SYSTEM_CREATED_FIELD; @@ -19,7 +19,6 @@ import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.AspectRetriever; -import com.linkedin.metadata.aspect.validation.StructuredPropertiesValidator; import com.linkedin.metadata.entity.EntityUtils; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; @@ -27,6 +26,7 @@ import com.linkedin.metadata.models.SearchScoreFieldSpec; import com.linkedin.metadata.models.SearchableFieldSpec; import com.linkedin.metadata.models.SearchableRefFieldSpec; +import com.linkedin.metadata.models.StructuredPropertyUtils; import com.linkedin.metadata.models.annotation.SearchableAnnotation.FieldType; import com.linkedin.metadata.models.extractor.FieldExtractor; import com.linkedin.metadata.models.registry.EntityRegistry; @@ -38,12 +38,15 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -109,6 +112,37 @@ public static ObjectNode withSystemCreated( return searchDocument; } + /** + * Handle object type UPSERTS where the new value to upsert removes a previous key. Only enabling + * for structured properties to start with i.e. + * + *

New => { "structuredProperties.foobar": "value1" } Old => { "structuredProperties.foobar": + * "value1" "structuredProperties.foobar2": "value2" } Expected => { + * "structuredProperties.foobar": "value1" "structuredProperties.foobar2": null } + * + * @param searchDocument new document + * @param previousSearchDocument previous document (if not present, no-op) + * @return searchDocument to upsert + */ + public static ObjectNode handleRemoveFields( + @Nonnull ObjectNode searchDocument, @Nullable ObjectNode previousSearchDocument) { + if (previousSearchDocument != null) { + Set documentFields = objectFieldsFilter(searchDocument.fieldNames()); + objectFieldsFilter(previousSearchDocument.fieldNames()).stream() + .filter(prevFieldName -> !documentFields.contains(prevFieldName)) + .forEach(removeFieldName -> searchDocument.set(removeFieldName, null)); + } + // no-op + return searchDocument; + } + + private static Set objectFieldsFilter(Iterator fieldNames) { + Iterable iterable = () -> fieldNames; + return StreamSupport.stream(iterable.spliterator(), false) + .filter(fieldName -> fieldName.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX)) + .collect(Collectors.toSet()); + } + public Optional transformAspect( @Nonnull OperationContext opContext, final @Nonnull Urn urn, @@ -388,25 +422,28 @@ private void setStructuredPropertiesSearchValue( .entrySet() .forEach( propertyEntry -> { - StructuredPropertyDefinition definition = - new StructuredPropertyDefinition( - definitions - .get(propertyEntry.getKey()) - .get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME) - .data()); + Optional definition = + Optional.ofNullable( + definitions + .get(propertyEntry.getKey()) + .get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)) + .map(def -> new StructuredPropertyDefinition(def.data())); + + LogicalValueType logicalValueType = + definition + .map(StructuredPropertyUtils::getLogicalValueType) + .orElse(LogicalValueType.UNKNOWN); String fieldName = String.join( ".", List.of( STRUCTURED_PROPERTY_MAPPING_FIELD, - sanitizeStructuredPropertyFQN(definition.getQualifiedName()))); + toElasticsearchFieldName( + propertyEntry.getKey(), definition.orElse(null)))); if (forDelete) { searchDocument.set(fieldName, JsonNodeFactory.instance.nullNode()); } else { - LogicalValueType logicalValueType = - StructuredPropertiesValidator.getLogicalValueType(definition.getValueType()); - ArrayNode arrayNode = JsonNodeFactory.instance.arrayNode(); propertyEntry @@ -487,7 +524,7 @@ private Optional getNodeForRef( final Object fieldValue, final FieldType fieldType) { EntityRegistry entityRegistry = opContext.getEntityRegistry(); - AspectRetriever aspectRetriever = opContext.getRetrieverContext().get().getAspectRetriever(); + AspectRetriever aspectRetriever = opContext.getAspectRetriever(); if (depth == 0) { if (fieldValue.toString().isEmpty()) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java index a6381b07c087c..e299dde62b184 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java @@ -2,6 +2,7 @@ import static com.linkedin.metadata.Constants.*; import static com.linkedin.metadata.models.annotation.SearchableAnnotation.OBJECT_FIELD_TYPES; +import static com.linkedin.metadata.search.elasticsearch.indexbuilder.MappingsBuilder.SUBFIELDS; import static com.linkedin.metadata.search.elasticsearch.query.request.SearchFieldConfig.KEYWORD_FIELDS; import static com.linkedin.metadata.search.elasticsearch.query.request.SearchFieldConfig.PATH_HIERARCHY_FIELDS; import static com.linkedin.metadata.search.utils.SearchUtils.isUrn; @@ -50,7 +51,6 @@ /** TODO: Add more robust unit tests for this critical class. */ @Slf4j public class ESUtils { - private static final String DEFAULT_SEARCH_RESULTS_SORT_BY_FIELD = "urn"; public static final String KEYWORD_ANALYZER = "keyword"; public static final String KEYWORD_SUFFIX = ".keyword"; @@ -140,7 +140,7 @@ public static BoolQueryBuilder buildFilterQuery( @Nullable Filter filter, boolean isTimeseries, final Map> searchableFieldTypes, - @Nonnull AspectRetriever aspectRetriever) { + @Nullable AspectRetriever aspectRetriever) { BoolQueryBuilder finalQueryBuilder = QueryBuilders.boolQuery(); if (filter == null) { return finalQueryBuilder; @@ -155,7 +155,8 @@ public static BoolQueryBuilder buildFilterQuery( .forEach( or -> finalQueryBuilder.should( - ESUtils.buildConjunctiveFilterQuery(or, isTimeseries, searchableFieldTypes))); + ESUtils.buildConjunctiveFilterQuery( + or, isTimeseries, searchableFieldTypes, aspectRetriever))); // The default is not always 1 (ensure consistent default) finalQueryBuilder.minimumShouldMatch(1); } else if (filter.getCriteria() != null) { @@ -170,7 +171,8 @@ public static BoolQueryBuilder buildFilterQuery( || criterion.hasValues() || criterion.getCondition() == Condition.IS_NULL) { andQueryBuilder.must( - getQueryBuilderFromCriterion(criterion, isTimeseries, searchableFieldTypes)); + getQueryBuilderFromCriterion( + criterion, isTimeseries, searchableFieldTypes, aspectRetriever)); } }); finalQueryBuilder.should(andQueryBuilder); @@ -184,7 +186,8 @@ public static BoolQueryBuilder buildFilterQuery( public static BoolQueryBuilder buildConjunctiveFilterQuery( @Nonnull ConjunctiveCriterion conjunctiveCriterion, boolean isTimeseries, - Map> searchableFieldTypes) { + Map> searchableFieldTypes, + @Nullable AspectRetriever aspectRetriever) { final BoolQueryBuilder andQueryBuilder = new BoolQueryBuilder(); conjunctiveCriterion .getAnd() @@ -196,10 +199,12 @@ public static BoolQueryBuilder buildConjunctiveFilterQuery( if (!criterion.isNegated()) { // `filter` instead of `must` (enables caching and bypasses scoring) andQueryBuilder.filter( - getQueryBuilderFromCriterion(criterion, isTimeseries, searchableFieldTypes)); + getQueryBuilderFromCriterion( + criterion, isTimeseries, searchableFieldTypes, aspectRetriever)); } else { andQueryBuilder.mustNot( - getQueryBuilderFromCriterion(criterion, isTimeseries, searchableFieldTypes)); + getQueryBuilderFromCriterion( + criterion, isTimeseries, searchableFieldTypes, aspectRetriever)); } } }); @@ -237,11 +242,9 @@ public static BoolQueryBuilder buildConjunctiveFilterQuery( public static QueryBuilder getQueryBuilderFromCriterion( @Nonnull final Criterion criterion, boolean isTimeseries, - final Map> searchableFieldTypes) { - final String fieldName = toFacetField(criterion.getField()); - if (fieldName.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD)) { - criterion.setField(fieldName); - } + final Map> searchableFieldTypes, + @Nullable AspectRetriever aspectRetriever) { + final String fieldName = toParentField(criterion.getField(), aspectRetriever); /* * Check the field-name for a "sibling" field, or one which should ALWAYS @@ -256,11 +259,11 @@ public static QueryBuilder getQueryBuilderFromCriterion( if (maybeFieldToExpand.isPresent()) { return getQueryBuilderFromCriterionForFieldToExpand( - maybeFieldToExpand.get(), criterion, isTimeseries, searchableFieldTypes); + maybeFieldToExpand.get(), criterion, isTimeseries, searchableFieldTypes, aspectRetriever); } return getQueryBuilderFromCriterionForSingleField( - criterion, isTimeseries, searchableFieldTypes); + criterion, isTimeseries, searchableFieldTypes, criterion.getField(), aspectRetriever); } public static String getElasticTypeForFieldType(SearchableAnnotation.FieldType fieldType) { @@ -398,28 +401,64 @@ public static String escapeReservedCharacters(@Nonnull String input) { return input; } + /** + * Resolve structured property field, or normal field, and strip subfields + * + * @param filterField name of the field used in the filter request + * @param aspectRetriever aspect retriever, used if structured property + * @return normalized field name without subfields + */ @Nonnull - public static String toFacetField(@Nonnull final String filterField) { - String fieldName = filterField; - if (fieldName.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD + ".")) { - String fqn = fieldName.substring(STRUCTURED_PROPERTY_MAPPING_FIELD.length() + 1); - fieldName = - STRUCTURED_PROPERTY_MAPPING_FIELD - + "." - + StructuredPropertyUtils.sanitizeStructuredPropertyFQN(fqn); + public static String toParentField( + @Nonnull final String filterField, @Nullable final AspectRetriever aspectRetriever) { + String fieldName = + StructuredPropertyUtils.lookupDefinitionFromFilterOrFacetName(filterField, aspectRetriever) + .map( + urnDefinition -> + STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX + + StructuredPropertyUtils.toElasticsearchFieldName( + urnDefinition.getFirst(), urnDefinition.getSecond())) + .orElse(filterField); + + for (String subfield : SUBFIELDS) { + String SUFFIX = "." + subfield; + if (filterField.endsWith(SUFFIX)) { + return fieldName.replace(SUFFIX, ""); + } } - return fieldName.replace(ESUtils.KEYWORD_SUFFIX, ""); + + return fieldName; } + /** + * Return resolved structured property field, normal field, or subfield which is of type `keyword` + * + * @param filterField the field name used in the filter + * @param skipKeywordSuffix prevent use of `keyword` subfield, useful when parent field is known + * or always `keyword` + * @param aspectRetriever aspect retriever, used if structured property field + * @return the preferred field to use for `keyword` queries + */ @Nonnull public static String toKeywordField( - @Nonnull final String filterField, final boolean skipKeywordSuffix) { + @Nonnull final String filterField, + final boolean skipKeywordSuffix, + @Nullable final AspectRetriever aspectRetriever) { + String fieldName = + StructuredPropertyUtils.lookupDefinitionFromFilterOrFacetName(filterField, aspectRetriever) + .map( + urnDefinition -> + STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX + + StructuredPropertyUtils.toElasticsearchFieldName( + urnDefinition.getFirst(), urnDefinition.getSecond())) + .orElse(filterField); + return skipKeywordSuffix - || KEYWORD_FIELDS.contains(filterField) - || PATH_HIERARCHY_FIELDS.contains(filterField) - || filterField.contains(".") - ? filterField - : filterField + ESUtils.KEYWORD_SUFFIX; + || KEYWORD_FIELDS.contains(fieldName) + || PATH_HIERARCHY_FIELDS.contains(fieldName) + || SUBFIELDS.stream().anyMatch(subfield -> fieldName.endsWith("." + subfield)) + ? fieldName + : fieldName + ESUtils.KEYWORD_SUFFIX; } public static RequestOptions buildReindexTaskRequestOptions( @@ -464,7 +503,8 @@ private static QueryBuilder getQueryBuilderFromCriterionForFieldToExpand( @Nonnull final List fields, @Nonnull final Criterion criterion, final boolean isTimeseries, - final Map> searchableFieldTypes) { + final Map> searchableFieldTypes, + @Nonnull AspectRetriever aspectRetriever) { final BoolQueryBuilder orQueryBuilder = new BoolQueryBuilder(); for (String field : fields) { Criterion criterionToQuery = new Criterion(); @@ -476,10 +516,11 @@ private static QueryBuilder getQueryBuilderFromCriterionForFieldToExpand( if (criterion.hasValue()) { criterionToQuery.setValue(criterion.getValue()); } - criterionToQuery.setField(toKeywordField(field, isTimeseries)); + criterionToQuery.setField(toKeywordField(field, isTimeseries, aspectRetriever)); orQueryBuilder.should( getQueryBuilderFromCriterionForSingleField( - criterionToQuery, isTimeseries, searchableFieldTypes)); + criterionToQuery, isTimeseries, searchableFieldTypes, null, aspectRetriever) + .queryName(field)); } return orQueryBuilder; } @@ -488,40 +529,49 @@ private static QueryBuilder getQueryBuilderFromCriterionForFieldToExpand( private static QueryBuilder getQueryBuilderFromCriterionForSingleField( @Nonnull Criterion criterion, boolean isTimeseries, - final Map> searchableFieldTypes) { + final Map> searchableFieldTypes, + @Nullable String queryName, + @Nonnull AspectRetriever aspectRetriever) { final Condition condition = criterion.getCondition(); - final String fieldName = toFacetField(criterion.getField()); + final String fieldName = toParentField(criterion.getField(), aspectRetriever); if (condition == Condition.IS_NULL) { return QueryBuilders.boolQuery() - .mustNot(QueryBuilders.existsQuery(criterion.getField())) - .queryName(fieldName); + .mustNot(QueryBuilders.existsQuery(fieldName)) + .queryName(queryName != null ? queryName : fieldName); } else if (condition == Condition.EXISTS) { return QueryBuilders.boolQuery() - .must(QueryBuilders.existsQuery(criterion.getField())) - .queryName(fieldName); + .must(QueryBuilders.existsQuery(fieldName)) + .queryName(queryName != null ? queryName : fieldName); } else if (criterion.hasValues() || criterion.hasValue()) { if (condition == Condition.EQUAL) { return buildEqualsConditionFromCriterion( - fieldName, criterion, isTimeseries, searchableFieldTypes); + fieldName, criterion, isTimeseries, searchableFieldTypes, aspectRetriever) + .queryName(queryName != null ? queryName : fieldName); } else if (RANGE_QUERY_CONDITIONS.contains(condition)) { return buildRangeQueryFromCriterion( - criterion, fieldName, searchableFieldTypes, condition, isTimeseries); + criterion, + fieldName, + searchableFieldTypes, + condition, + isTimeseries, + aspectRetriever) + .queryName(queryName != null ? queryName : fieldName); } else if (condition == Condition.CONTAIN) { return QueryBuilders.wildcardQuery( - toKeywordField(criterion.getField(), isTimeseries), + toKeywordField(criterion.getField(), isTimeseries, aspectRetriever), "*" + ESUtils.escapeReservedCharacters(criterion.getValue().trim()) + "*") - .queryName(fieldName); + .queryName(queryName != null ? queryName : fieldName); } else if (condition == Condition.START_WITH) { return QueryBuilders.wildcardQuery( - toKeywordField(criterion.getField(), isTimeseries), + toKeywordField(criterion.getField(), isTimeseries, aspectRetriever), ESUtils.escapeReservedCharacters(criterion.getValue().trim()) + "*") - .queryName(fieldName); + .queryName(queryName != null ? queryName : fieldName); } else if (condition == Condition.END_WITH) { return QueryBuilders.wildcardQuery( - toKeywordField(criterion.getField(), isTimeseries), + toKeywordField(criterion.getField(), isTimeseries, aspectRetriever), "*" + ESUtils.escapeReservedCharacters(criterion.getValue().trim())) - .queryName(fieldName); + .queryName(queryName != null ? queryName : fieldName); } } throw new UnsupportedOperationException("Unsupported condition: " + condition); @@ -531,20 +581,21 @@ private static QueryBuilder buildEqualsConditionFromCriterion( @Nonnull final String fieldName, @Nonnull final Criterion criterion, final boolean isTimeseries, - final Map> searchableFieldTypes) { + final Map> searchableFieldTypes, + @Nonnull AspectRetriever aspectRetriever) { /* * If the newer 'values' field of Criterion.pdl is set, then we * handle using the following code to allow multi-match. */ if (!criterion.getValues().isEmpty()) { return buildEqualsConditionFromCriterionWithValues( - fieldName, criterion, isTimeseries, searchableFieldTypes); + fieldName, criterion, isTimeseries, searchableFieldTypes, aspectRetriever); } /* * Otherwise, we are likely using the deprecated 'value' field. * We handle using the legacy code path below. */ - return buildEqualsFromCriterionWithValue(fieldName, criterion, isTimeseries); + return buildEqualsFromCriterionWithValue(fieldName, criterion, isTimeseries, aspectRetriever); } /** @@ -555,8 +606,9 @@ private static QueryBuilder buildEqualsConditionFromCriterionWithValues( @Nonnull final String fieldName, @Nonnull final Criterion criterion, final boolean isTimeseries, - final Map> searchableFieldTypes) { - Set fieldTypes = getFieldTypes(searchableFieldTypes, fieldName); + final Map> searchableFieldTypes, + @Nonnull AspectRetriever aspectRetriever) { + Set fieldTypes = getFieldTypes(searchableFieldTypes, fieldName, aspectRetriever); if (fieldTypes.size() > 1) { log.warn( "Multiple field types for field name {}, determining best fit for set: {}", @@ -576,21 +628,32 @@ private static QueryBuilder buildEqualsConditionFromCriterionWithValues( return QueryBuilders.termsQuery(fieldName, doubleValues).queryName(fieldName); } return QueryBuilders.termsQuery( - toKeywordField(criterion.getField(), isTimeseries), criterion.getValues()) + toKeywordField(criterion.getField(), isTimeseries, aspectRetriever), + criterion.getValues()) .queryName(fieldName); } private static Set getFieldTypes( - Map> searchableFields, String fieldName) { - Set fieldTypes = - searchableFields.getOrDefault(fieldName, Collections.emptySet()); - Set finalFieldTypes = - fieldTypes.stream().map(ESUtils::getElasticTypeForFieldType).collect(Collectors.toSet()); - if (fieldTypes.size() > 1) { + Map> searchableFields, + String fieldName, + @Nullable AspectRetriever aspectRetriever) { + + final Set finalFieldTypes; + if (fieldName.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX)) { + finalFieldTypes = + StructuredPropertyUtils.toElasticsearchFieldType(fieldName, aspectRetriever); + } else { + Set fieldTypes = + searchableFields.getOrDefault(fieldName, Collections.emptySet()); + finalFieldTypes = + fieldTypes.stream().map(ESUtils::getElasticTypeForFieldType).collect(Collectors.toSet()); + } + + if (finalFieldTypes.size() > 1) { log.warn( "Multiple field types for field name {}, determining best fit for set: {}", fieldName, - fieldTypes); + finalFieldTypes); } return finalFieldTypes; } @@ -600,8 +663,9 @@ private static RangeQueryBuilder buildRangeQueryFromCriterion( String fieldName, Map> searchableFieldTypes, Condition condition, - boolean isTimeseries) { - Set fieldTypes = getFieldTypes(searchableFieldTypes, fieldName); + boolean isTimeseries, + AspectRetriever aspectRetriever) { + Set fieldTypes = getFieldTypes(searchableFieldTypes, fieldName, aspectRetriever); // Determine criterion value, range query only accepts single value so take first value in // values if multiple @@ -624,7 +688,7 @@ private static RangeQueryBuilder buildRangeQueryFromCriterion( documentFieldName = fieldName; } else { criterionValue = criterionValueString; - documentFieldName = toKeywordField(fieldName, isTimeseries); + documentFieldName = toKeywordField(fieldName, isTimeseries, aspectRetriever); } // Set up QueryBuilder based on condition @@ -653,12 +717,14 @@ private static RangeQueryBuilder buildRangeQueryFromCriterion( private static QueryBuilder buildEqualsFromCriterionWithValue( @Nonnull final String fieldName, @Nonnull final Criterion criterion, - final boolean isTimeseries) { + final boolean isTimeseries, + @Nonnull AspectRetriever aspectRetriever) { // If the value is an URN style value, then we do not attempt to split it by comma (for obvious // reasons) if (isUrn(criterion.getValue())) { return QueryBuilders.matchQuery( - toKeywordField(criterion.getField(), isTimeseries), criterion.getValue().trim()) + toKeywordField(criterion.getField(), isTimeseries, aspectRetriever), + criterion.getValue().trim()) .queryName(fieldName) .analyzer(KEYWORD_ANALYZER); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/BusinessAttributeUpdateHookService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/BusinessAttributeUpdateHookService.java index b1a74b9c09d35..ad2825ead3d0d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/service/BusinessAttributeUpdateHookService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/BusinessAttributeUpdateHookService.java @@ -22,12 +22,13 @@ import com.linkedin.mxe.PlatformEvent; import com.linkedin.platform.event.v1.EntityChangeEvent; import io.datahubproject.metadata.context.OperationContext; -import java.util.Arrays; -import java.util.Map; -import java.util.Set; -import java.util.function.BiConsumer; +import java.util.*; +import java.util.concurrent.*; import java.util.stream.Collectors; +import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.time.StopWatch; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.lang.NonNull; import org.springframework.lang.Nullable; @@ -41,18 +42,25 @@ public class BusinessAttributeUpdateHookService { private final UpdateIndicesService updateIndicesService; private final int relatedEntitiesCount; private final int getRelatedEntitiesBatchSize; - + private ExecutorService executor; public static final String TAG = "TAG"; public static final String GLOSSARY_TERM = "GLOSSARY_TERM"; public static final String DOCUMENTATION = "DOCUMENTATION"; + private final int threadCount; + private final int AWAIT_TERMINATION_TIME = 10; + private final int keepAlive; public BusinessAttributeUpdateHookService( @NonNull UpdateIndicesService updateIndicesService, @NonNull @Value("${businessAttribute.fetchRelatedEntitiesCount}") int relatedEntitiesCount, - @NonNull @Value("${businessAttribute.fetchRelatedEntitiesBatchSize}") int relatedBatchSize) { + @NonNull @Value("${businessAttribute.fetchRelatedEntitiesBatchSize}") int relatedBatchSize, + @NonNull @Value("${businessAttribute.threadCount}") int threadCount, + @NonNull @Value("${businessAttribute.keepAliveTime}") int keepAlive) { this.updateIndicesService = updateIndicesService; this.relatedEntitiesCount = relatedEntitiesCount; this.getRelatedEntitiesBatchSize = relatedBatchSize; + this.threadCount = threadCount; + this.keepAlive = keepAlive; } public void handleChangeEvent( @@ -61,38 +69,51 @@ public void handleChangeEvent( GenericRecordUtils.deserializePayload( event.getPayload().getValue(), EntityChangeEvent.class); + executor = businessAttributePropagationWorkerPool(threadCount, keepAlive); + if (!entityChangeEvent.getEntityType().equals(Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME)) { - log.info("Skipping MCL event for entity:" + entityChangeEvent.getEntityType()); return; } final Set businessAttributeCategories = ImmutableSet.of(TAG, GLOSSARY_TERM, DOCUMENTATION); if (!businessAttributeCategories.contains(entityChangeEvent.getCategory())) { - log.info("Skipping MCL event for category: " + entityChangeEvent.getCategory()); return; } Urn urn = entityChangeEvent.getEntityUrn(); - log.info("Business Attribute update hook invoked for urn :" + urn); + log.info("Business Attribute update hook invoked for urn : {}", urn); fetchRelatedEntities( opContext, urn, - (batch, batchNumber) -> processBatch(opContext, batch, batchNumber), + (batch, batchNumber, entityKey) -> processBatch(opContext, batch, batchNumber, entityKey), null, 0, 1); + + executor.shutdown(); + try { + if (!executor.awaitTermination(AWAIT_TERMINATION_TIME, TimeUnit.MINUTES)) { + executor.shutdownNow(); // Cancel currently executing tasks + if (!executor.awaitTermination(AWAIT_TERMINATION_TIME, TimeUnit.MINUTES)) + log.error("Business Attribute Propagation Executor is not terminating"); + } + } catch (InterruptedException ie) { + executor.shutdownNow(); + } } private void fetchRelatedEntities( @NonNull final OperationContext opContext, @NonNull final Urn urn, - @NonNull final BiConsumer resultConsumer, + @NonNull + final TriFunction> + resultFunction, @Nullable String scrollId, int consumedEntityCount, int batchNumber) { GraphRetriever graph = opContext.getRetrieverContext().get().getGraphRetriever(); - + final ArrayList> futureList = new ArrayList<>(); RelatedEntitiesScrollResult result = graph.scrollRelatedEntities( null, @@ -106,52 +127,143 @@ private void fetchRelatedEntities( getRelatedEntitiesBatchSize, null, null); - resultConsumer.accept(result, batchNumber); + + futureList.add( + executor.submit(resultFunction.apply(result, batchNumber, urn.getEntityKey().toString()))); + consumedEntityCount = consumedEntityCount + result.getEntities().size(); if (result.getScrollId() != null && consumedEntityCount < relatedEntitiesCount) { batchNumber = batchNumber + 1; fetchRelatedEntities( - opContext, urn, resultConsumer, result.getScrollId(), consumedEntityCount, batchNumber); + opContext, urn, resultFunction, result.getScrollId(), consumedEntityCount, batchNumber); + } + + for (Future future : futureList) { + try { + ExecutionResult futureResult = future.get(); + if (futureResult.getException() != null) { + log.error( + "Batch {} for BA:{} is failed with exception", + futureResult.getBatchNumber(), + futureResult.getEntityKey(), + futureResult.getException()); + } else { + log.info(futureResult.getResult()); + } + } catch (InterruptedException | ExecutionException e) { + log.error("Business Attribute Propagation Parallel Processing Exception", e); + } } + futureList.clear(); } - private void processBatch( + private Callable processBatch( @NonNull OperationContext opContext, @NonNull RelatedEntitiesScrollResult batch, - int batchNumber) { - AspectRetriever aspectRetriever = opContext.getRetrieverContext().get().getAspectRetriever(); - log.info("BA Update Batch {} started", batchNumber); - Set entityUrns = - batch.getEntities().stream() - .map(RelatedEntity::getUrn) - .map(UrnUtils::getUrn) - .collect(Collectors.toSet()); - - Map> entityAspectMap = - aspectRetriever.getLatestAspectObjects( - entityUrns, Set.of(Constants.BUSINESS_ATTRIBUTE_ASPECT)); - - entityAspectMap.entrySet().stream() - .filter(entry -> entry.getValue().containsKey(Constants.BUSINESS_ATTRIBUTE_ASPECT)) - .forEach( - entry -> { - final Urn entityUrn = entry.getKey(); - final Aspect aspect = entry.getValue().get(Constants.BUSINESS_ATTRIBUTE_ASPECT); - - updateIndicesService.handleChangeEvent( - opContext, - PegasusUtils.constructMCL( - null, - Constants.SCHEMA_FIELD_ENTITY_NAME, - entityUrn, - ChangeType.UPSERT, - Constants.BUSINESS_ATTRIBUTE_ASPECT, - opContext.getAuditStamp(), - new BusinessAttributes(aspect.data()), - null, - null, - null)); - }); - log.info("BA Update Batch {} completed", batchNumber); + int batchNumber, + String entityKey) { + return () -> { + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + AspectRetriever aspectRetriever = opContext.getAspectRetrieverOpt().get(); + log.info("Batch {} for BA:{} started", batchNumber, entityKey); + ExecutionResult executionResult = new ExecutionResult(); + executionResult.setBatchNumber(batchNumber); + executionResult.setEntityKey(entityKey); + try { + Set entityUrns = + batch.getEntities().stream() + .map(RelatedEntity::getUrn) + .map(UrnUtils::getUrn) + .collect(Collectors.toSet()); + + Map> entityAspectMap = + aspectRetriever.getLatestAspectObjects( + entityUrns, Set.of(Constants.BUSINESS_ATTRIBUTE_ASPECT)); + + entityAspectMap.entrySet().stream() + .filter(entry -> entry.getValue().containsKey(Constants.BUSINESS_ATTRIBUTE_ASPECT)) + .forEach( + entry -> { + final Urn entityUrn = entry.getKey(); + final Aspect aspect = entry.getValue().get(Constants.BUSINESS_ATTRIBUTE_ASPECT); + updateIndicesService.handleChangeEvent( + opContext, + PegasusUtils.constructMCL( + null, + Constants.SCHEMA_FIELD_ENTITY_NAME, + entityUrn, + ChangeType.UPSERT, + Constants.BUSINESS_ATTRIBUTE_ASPECT, + opContext.getAuditStamp(), + new BusinessAttributes(aspect.data()), + null, + null, + null)); + }); + stopWatch.stop(); + String result = + String.format( + "Batch %s for BA:%s is completed in %s", + batchNumber, entityKey, TimeAgo.toDuration(stopWatch.getTime())) + .toString(); + executionResult.setResult(result); + } catch (Exception e) { + executionResult.setException(e); + } + return executionResult; + }; + } + + private ExecutorService businessAttributePropagationWorkerPool(int numThreads, int keepAlive) { + numThreads = numThreads < 0 ? Runtime.getRuntime().availableProcessors() * 2 : numThreads; + return new ThreadPoolExecutor( + numThreads, numThreads, keepAlive, TimeUnit.SECONDS, new LinkedBlockingQueue()); + } + + @FunctionalInterface + private interface TriFunction { + R apply(T t, U u, V v); + } + + @Data + private class ExecutionResult { + String result; + Throwable exception; + int batchNumber; + String entityKey; + } + + private static final class TimeAgo { + private static final List times = + Arrays.asList( + TimeUnit.DAYS.toMillis(365), + TimeUnit.DAYS.toMillis(30), + TimeUnit.DAYS.toMillis(1), + TimeUnit.HOURS.toMillis(1), + TimeUnit.MINUTES.toMillis(1), + TimeUnit.SECONDS.toMillis(1), + TimeUnit.MILLISECONDS.toMillis(1)); + private static final List timesString = + Arrays.asList("year", "month", "day", "hour", "minute", "second", "milliseconds"); + + private static String toDuration(long duration) { + + StringBuffer res = new StringBuffer(); + for (int i = 0; i < times.size(); i++) { + Long current = times.get(i); + long temp = duration / current; + if (temp > 0) { + res.append(temp) + .append(" ") + .append(timesString.get(i)) + .append(temp != 1 ? "s" : StringUtils.EMPTY) + .append(" "); + } + duration = duration % current; + } + if (StringUtils.EMPTY.equals(res.toString())) return "0 seconds ago"; + else return res.toString(); + } } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java index da35c9e0b0784..dff0a99a142b7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; import com.linkedin.common.InputField; @@ -130,8 +131,7 @@ public void handleChangeEvent( @Nonnull OperationContext opContext, @Nonnull final MetadataChangeLog event) { try { MCLItemImpl batch = - MCLItemImpl.builder() - .build(event, opContext.getRetrieverContext().get().getAspectRetriever()); + MCLItemImpl.builder().build(event, opContext.getAspectRetrieverOpt().get()); Stream sideEffects = AspectsBatch.applyMCLSideEffects(List.of(batch), opContext.getRetrieverContext().get()); @@ -187,7 +187,7 @@ private void handleUpdateChangeEvent( } // Step 1. Handle StructuredProperties Index Mapping changes - updateIndexMappings(entitySpec, aspectSpec, aspect, previousAspect); + updateIndexMappings(urn, entitySpec, aspectSpec, aspect, previousAspect); // Step 2. For all aspects, attempt to update Search updateSearchService(opContext, event); @@ -206,6 +206,7 @@ private void handleUpdateChangeEvent( } public void updateIndexMappings( + @Nonnull Urn urn, EntitySpec entitySpec, AspectSpec aspectSpec, RecordTemplate newValue, @@ -228,7 +229,7 @@ public void updateIndexMappings( if (newDefinition.getEntityTypes().size() > 0) { _entityIndexBuilders - .buildReindexConfigsWithNewStructProp(newDefinition) + .buildReindexConfigsWithNewStructProp(urn, newDefinition) .forEach( reindexState -> { try { @@ -526,8 +527,8 @@ private void updateSearchService(@Nonnull OperationContext opContext, MCLItem ev RecordTemplate previousAspect = event.getPreviousRecordTemplate(); String entityName = event.getEntitySpec().getName(); - Optional searchDocument; - Optional previousSearchDocument = Optional.empty(); + Optional searchDocument; + Optional previousSearchDocument = Optional.empty(); try { searchDocument = _searchDocumentTransformer @@ -539,8 +540,7 @@ private void updateSearchService(@Nonnull OperationContext opContext, MCLItem ev event.getChangeType(), event.getEntitySpec(), aspectSpec, - event.getAuditStamp())) - .map(Objects::toString); + event.getAuditStamp())); } catch (Exception e) { log.error( "Error in getting documents from aspect: {} for aspect {}", e, aspectSpec.getName()); @@ -557,7 +557,6 @@ private void updateSearchService(@Nonnull OperationContext opContext, MCLItem ev return; } - String searchDocumentValue = searchDocument.get(); if (_searchDiffMode && (systemMetadata == null || systemMetadata.getProperties() == null @@ -565,9 +564,8 @@ private void updateSearchService(@Nonnull OperationContext opContext, MCLItem ev if (previousAspect != null) { try { previousSearchDocument = - _searchDocumentTransformer - .transformAspect(opContext, urn, previousAspect, aspectSpec, false) - .map(Objects::toString); + _searchDocumentTransformer.transformAspect( + opContext, urn, previousAspect, aspectSpec, false); } catch (Exception e) { log.error( "Error in getting documents from previous aspect state: {} for aspect {}, continuing without diffing.", @@ -577,15 +575,19 @@ private void updateSearchService(@Nonnull OperationContext opContext, MCLItem ev } if (previousSearchDocument.isPresent()) { - String previousSearchDocumentValue = previousSearchDocument.get(); - if (searchDocumentValue.equals(previousSearchDocumentValue)) { + if (searchDocument.get().toString().equals(previousSearchDocument.get().toString())) { // No changes to search document, skip writing no-op update return; } } } - _entitySearchService.upsertDocument(opContext, entityName, searchDocument.get(), docId.get()); + String finalDocument = + SearchDocumentTransformer.handleRemoveFields( + searchDocument.get(), previousSearchDocument.orElse(null)) + .toString(); + + _entitySearchService.upsertDocument(opContext, entityName, finalDocument, docId.get()); } /** Process snapshot and update time-series index */ diff --git a/metadata-io/src/main/java/com/linkedin/metadata/shared/ElasticSearchIndexed.java b/metadata-io/src/main/java/com/linkedin/metadata/shared/ElasticSearchIndexed.java index e894558e3d1af..51d2cee06730c 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/shared/ElasticSearchIndexed.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/shared/ElasticSearchIndexed.java @@ -1,31 +1,27 @@ package com.linkedin.metadata.shared; +import com.linkedin.common.urn.Urn; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ReindexConfig; import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.util.Pair; import java.io.IOException; import java.util.Collection; import java.util.List; public interface ElasticSearchIndexed { - /** - * The index configurations for the given service. - * - * @return List of reindex configurations - */ - List buildReindexConfigs() throws IOException; - /** * The index configurations for the given service with StructuredProperties applied. * * @param properties The structured properties to apply to the index mappings * @return List of reindex configurations */ - List buildReindexConfigsWithAllStructProps( - Collection properties) throws IOException; + List buildReindexConfigs( + Collection> properties) throws IOException; /** * Mirrors the service's functions which are expected to build/reindex as needed based on the * reindex configurations above */ - void reindexAll() throws IOException; + void reindexAll(Collection> properties) + throws IOException; } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/hooks/PropertyDefinitionDeleteSideEffect.java b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/hooks/PropertyDefinitionDeleteSideEffect.java new file mode 100644 index 0000000000000..41addbe197f27 --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/hooks/PropertyDefinitionDeleteSideEffect.java @@ -0,0 +1,201 @@ +package com.linkedin.metadata.structuredproperties.hooks; + +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_KEY_ASPECT_NAME; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.entity.Aspect; +import com.linkedin.metadata.aspect.RetrieverContext; +import com.linkedin.metadata.aspect.batch.ChangeMCP; +import com.linkedin.metadata.aspect.batch.MCLItem; +import com.linkedin.metadata.aspect.batch.MCPItem; +import com.linkedin.metadata.aspect.patch.GenericJsonPatch; +import com.linkedin.metadata.aspect.patch.PatchOperationType; +import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; +import com.linkedin.metadata.aspect.plugins.hooks.MCPSideEffect; +import com.linkedin.metadata.entity.SearchRetriever; +import com.linkedin.metadata.entity.ebean.batch.PatchItemImpl; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.StructuredPropertyUtils; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.ScrollResult; +import com.linkedin.structured.StructuredPropertyDefinition; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Getter +@Setter +@Accessors(chain = true) +public class PropertyDefinitionDeleteSideEffect extends MCPSideEffect { + public static final Integer SEARCH_SCROLL_SIZE = 1000; + @Nonnull private AspectPluginConfig config; + + @Override + protected Stream applyMCPSideEffect( + Collection changeMCPS, @Nonnull RetrieverContext retrieverContext) { + return Stream.of(); + } + + @Override + protected Stream postMCPSideEffect( + Collection mclItems, @Nonnull RetrieverContext retrieverContext) { + return mclItems.stream().flatMap(item -> generatePatchRemove(item, retrieverContext)); + } + + private static Stream generatePatchRemove( + MCLItem mclItem, @Nonnull RetrieverContext retrieverContext) { + + if (STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME.equals(mclItem.getAspectName())) { + return generatePatchMCPs( + mclItem.getUrn(), + mclItem.getPreviousAspect(StructuredPropertyDefinition.class), + mclItem.getAuditStamp(), + retrieverContext); + } else if (STRUCTURED_PROPERTY_KEY_ASPECT_NAME.equals(mclItem.getAspectName())) { + Aspect definitionAspect = + retrieverContext + .getAspectRetriever() + .getLatestAspectObject(mclItem.getUrn(), STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME); + return generatePatchMCPs( + mclItem.getUrn(), + definitionAspect == null + ? null + : new StructuredPropertyDefinition(definitionAspect.data()), + mclItem.getAuditStamp(), + retrieverContext); + } + log.warn( + "Expected either {} or {} aspects but got {}", + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, + STRUCTURED_PROPERTY_KEY_ASPECT_NAME, + mclItem.getAspectName()); + return Stream.empty(); + } + + private static Stream generatePatchMCPs( + Urn propertyUrn, + @Nullable StructuredPropertyDefinition definition, + @Nullable AuditStamp auditStamp, + @Nonnull RetrieverContext retrieverContext) { + EntityWithPropertyIterator iterator = + EntityWithPropertyIterator.builder() + .propertyUrn(propertyUrn) + .definition(definition) + .searchRetriever(retrieverContext.getSearchRetriever()) + .count(SEARCH_SCROLL_SIZE) + .build(); + return StreamSupport.stream( + Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false) + .flatMap( + scrollResult -> + scrollResult.getEntities().stream() + .map( + entity -> { + GenericJsonPatch.PatchOp patchOp = new GenericJsonPatch.PatchOp(); + patchOp.setOp(PatchOperationType.REMOVE.getValue()); + patchOp.setPath(String.format("/properties/%s", propertyUrn.toString())); + + EntitySpec entitySpec = + retrieverContext + .getAspectRetriever() + .getEntityRegistry() + .getEntitySpec(entity.getEntity().getEntityType()); + return PatchItemImpl.builder() + .urn(entity.getEntity()) + .entitySpec(entitySpec) + .aspectName(STRUCTURED_PROPERTIES_ASPECT_NAME) + .aspectSpec( + entitySpec.getAspectSpec(STRUCTURED_PROPERTIES_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys( + Map.of("properties", List.of("propertyUrn"))) + .patch(List.of(patchOp)) + .build() + .getJsonPatch()) + .auditStamp(auditStamp) + .build(retrieverContext.getAspectRetriever().getEntityRegistry()); + })); + } + + /** + * Fetches pages of entity urns which have a value for the given structured property definition + */ + @Builder + public static class EntityWithPropertyIterator implements Iterator { + @Nonnull private final Urn propertyUrn; + @Nullable private final StructuredPropertyDefinition definition; + @Nonnull private final SearchRetriever searchRetriever; + private int count; + @Builder.Default private String scrollId = null; + @Builder.Default private boolean started = false; + + private List getEntities() { + if (definition != null && definition.getEntityTypes() != null) { + return definition.getEntityTypes().stream() + .map(StructuredPropertyUtils::getValueTypeId) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + private Filter getFilter() { + Filter propertyFilter = new Filter(); + final ConjunctiveCriterionArray disjunction = new ConjunctiveCriterionArray(); + final ConjunctiveCriterion conjunction = new ConjunctiveCriterion(); + final CriterionArray andCriterion = new CriterionArray(); + + final Criterion propertyExistsCriterion = new Criterion(); + // Cannot rely on automatic field name since the definition is deleted + propertyExistsCriterion.setField( + STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX + + StructuredPropertyUtils.toElasticsearchFieldName(propertyUrn, definition)); + propertyExistsCriterion.setCondition(Condition.EXISTS); + + andCriterion.add(propertyExistsCriterion); + conjunction.setAnd(andCriterion); + disjunction.add(conjunction); + propertyFilter.setOr(disjunction); + + return propertyFilter; + } + + @Override + public boolean hasNext() { + return !started || scrollId != null; + } + + @Override + public ScrollResult next() { + started = true; + ScrollResult result = searchRetriever.scroll(getEntities(), getFilter(), scrollId, count); + scrollId = result.getScrollId(); + return result; + } + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/hooks/StructuredPropertiesSoftDelete.java b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/hooks/StructuredPropertiesSoftDelete.java similarity index 96% rename from entity-registry/src/main/java/com/linkedin/metadata/aspect/hooks/StructuredPropertiesSoftDelete.java rename to metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/hooks/StructuredPropertiesSoftDelete.java index 5efb1e8aebb06..99b5832837675 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/hooks/StructuredPropertiesSoftDelete.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/hooks/StructuredPropertiesSoftDelete.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata.aspect.hooks; +package com.linkedin.metadata.structuredproperties.hooks; import com.linkedin.common.urn.Urn; import com.linkedin.metadata.aspect.ReadItem; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/PropertyDefinitionValidator.java b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/PropertyDefinitionValidator.java similarity index 68% rename from entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/PropertyDefinitionValidator.java rename to metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/PropertyDefinitionValidator.java index a4efc38d16082..ae5472af622ad 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/PropertyDefinitionValidator.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/PropertyDefinitionValidator.java @@ -1,20 +1,20 @@ -package com.linkedin.metadata.aspect.validation; +package com.linkedin.metadata.structuredproperties.validation; import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME; import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; -import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME; import static com.linkedin.structured.PropertyCardinality.*; import com.google.common.collect.ImmutableSet; import com.linkedin.common.Status; import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.GetMode; import com.linkedin.entity.Aspect; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.RetrieverContext; import com.linkedin.metadata.aspect.batch.BatchItem; import com.linkedin.metadata.aspect.batch.ChangeMCP; +import com.linkedin.metadata.aspect.batch.MCPItem; import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator; import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; @@ -25,12 +25,13 @@ import java.util.Collection; import java.util.Collections; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import lombok.Getter; import lombok.Setter; import lombok.experimental.Accessors; @@ -52,25 +53,7 @@ public class PropertyDefinitionValidator extends AspectPayloadValidator { protected Stream validateProposedAspects( @Nonnull Collection mcpItems, @Nonnull RetrieverContext retrieverContext) { - final String entityKeyAspect = - retrieverContext - .getAspectRetriever() - .getEntityRegistry() - .getEntitySpec(STRUCTURED_PROPERTY_ENTITY_NAME) - .getKeyAspectName(); - - return mcpItems.stream() - .filter(i -> ChangeType.DELETE.equals(i.getChangeType())) - .map( - i -> { - if (ImmutableSet.of(entityKeyAspect, STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME) - .contains(i.getAspectSpec().getName())) { - return AspectValidationException.forItem( - i, "Hard delete of Structured Property Definitions is not supported."); - } - return null; - }) - .filter(Objects::nonNull); + return Stream.empty(); } @Override @@ -78,10 +61,7 @@ protected Stream validatePreCommitAspects( @Nonnull Collection changeMCPs, @Nonnull RetrieverContext retrieverContext) { return validateDefinitionUpserts( changeMCPs.stream() - .filter( - i -> - ChangeType.UPSERT.equals(i.getChangeType()) - && STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME.equals(i.getAspectName())) + .filter(i -> STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME.equals(i.getAspectName())) .collect(Collectors.toList()), retrieverContext); } @@ -105,19 +85,24 @@ public static Stream validateDefinitionUpserts( "Cannot mutate a soft deleted Structured Property Definition") .ifPresent(exceptions::addException); + final StructuredPropertyDefinition newDefinition = + item.getAspect(StructuredPropertyDefinition.class); + + versionFormatCheck(item, newDefinition.getVersion()).ifPresent(exceptions::addException); + if (item.getPreviousSystemAspect() != null) { StructuredPropertyDefinition previousDefinition = item.getPreviousSystemAspect().getAspect(StructuredPropertyDefinition.class); - StructuredPropertyDefinition newDefinition = - item.getAspect(StructuredPropertyDefinition.class); - if (!newDefinition.getValueType().equals(previousDefinition.getValueType())) { + if (!newDefinition.getValueType().equals(previousDefinition.getValueType()) + && !allowBreakingWithVersion(previousDefinition, newDefinition, item, exceptions)) { exceptions.addException( item, "Value type cannot be changed as this is a backwards incompatible change"); } if (newDefinition.getCardinality().equals(SINGLE) - && previousDefinition.getCardinality().equals(MULTIPLE)) { + && previousDefinition.getCardinality().equals(MULTIPLE) + && !allowBreakingWithVersion(previousDefinition, newDefinition, item, exceptions)) { exceptions.addException( item, "Property definition cardinality cannot be changed from MULTI to SINGLE"); } @@ -127,10 +112,12 @@ public static Stream validateDefinitionUpserts( } // Assure new definition has only added allowed values, not removed them if (newDefinition.getAllowedValues() != null) { - if (!previousDefinition.hasAllowedValues() - || previousDefinition.getAllowedValues() == null) { + if ((!previousDefinition.hasAllowedValues() + || previousDefinition.getAllowedValues() == null) + && !allowBreakingWithVersion(previousDefinition, newDefinition, item, exceptions)) { exceptions.addException(item, "Cannot restrict values that were previously allowed"); - } else { + } else if (!allowBreakingWithVersion( + previousDefinition, newDefinition, item, exceptions)) { Set newAllowedValues = newDefinition.getAllowedValues().stream() .map(PropertyValue::getValue) @@ -163,4 +150,46 @@ static Optional softDeleteCheck } return Optional.empty(); } + + /** + * Allow new version if monotonically increasing + * + * @param oldDefinition previous version + * @param newDefinition next version + * @return whether version increase should allow breaking change + */ + private static boolean allowBreakingWithVersion( + @Nonnull StructuredPropertyDefinition oldDefinition, + @Nonnull StructuredPropertyDefinition newDefinition, + @Nonnull ChangeMCP item, + @Nonnull ValidationExceptionCollection exceptions) { + final String oldVersion = oldDefinition.getVersion(GetMode.NULL); + final String newVersion = newDefinition.getVersion(GetMode.NULL); + + if (newVersion != null && newVersion.contains(".")) { + exceptions.addException( + item, + String.format("Invalid version `%s` cannot contain the `.` character.", newVersion)); + } + + if (oldVersion == null && newVersion != null) { + return true; + } else if (newVersion != null) { + return newVersion.compareToIgnoreCase(oldVersion) > 0; + } + return false; + } + + private static Pattern VERSION_REGEX = Pattern.compile("[0-9]{14}"); + + private static Optional versionFormatCheck( + MCPItem item, @Nullable String version) { + if (version != null && !VERSION_REGEX.matcher(version).matches()) { + return Optional.of( + AspectValidationException.forItem( + item, + String.format("Invalid version specified. Must match %s", VERSION_REGEX.toString()))); + } + return Optional.empty(); + } } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/StructuredPropertiesValidator.java b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/StructuredPropertiesValidator.java similarity index 94% rename from entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/StructuredPropertiesValidator.java rename to metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/StructuredPropertiesValidator.java index fcae6ca8cb71a..cdbe2eb95a15d 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/StructuredPropertiesValidator.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/StructuredPropertiesValidator.java @@ -1,7 +1,9 @@ -package com.linkedin.metadata.aspect.validation; +package com.linkedin.metadata.structuredproperties.validation; import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; -import static com.linkedin.metadata.aspect.validation.PropertyDefinitionValidator.softDeleteCheck; +import static com.linkedin.metadata.models.StructuredPropertyUtils.getLogicalValueType; +import static com.linkedin.metadata.models.StructuredPropertyUtils.getValueTypeId; +import static com.linkedin.metadata.structuredproperties.validation.PropertyDefinitionValidator.softDeleteCheck; import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; @@ -66,23 +68,6 @@ public class StructuredPropertiesValidator extends AspectPayloadValidator { LogicalValueType.DATE, LogicalValueType.URN)); - public static LogicalValueType getLogicalValueType(Urn valueType) { - String valueTypeId = getValueTypeId(valueType); - if (valueTypeId.equals("string")) { - return LogicalValueType.STRING; - } else if (valueTypeId.equals("date")) { - return LogicalValueType.DATE; - } else if (valueTypeId.equals("number")) { - return LogicalValueType.NUMBER; - } else if (valueTypeId.equals("urn")) { - return LogicalValueType.URN; - } else if (valueTypeId.equals("rich_text")) { - return LogicalValueType.RICH_TEXT; - } - - return LogicalValueType.UNKNOWN; - } - @Nonnull private AspectPluginConfig config; @Override @@ -132,7 +117,11 @@ public static Stream validateProposedUpserts( StructuredPropertyDefinition structuredPropertyDefinition = lookupPropertyDefinition(propertyUrn, allStructuredPropertiesAspects); if (structuredPropertyDefinition == null) { - exceptions.addException(i, "Unexpected null value found."); + exceptions.addException( + i, + String.format( + "Unexpected null value found for %s Structured Property Definition.", + propertyUrn)); } log.debug( @@ -237,7 +226,7 @@ private static Set validateStructuredPropertyUrns( for (BatchItem i : exceptions.successful(mcpItems)) { StructuredProperties structuredProperties = i.getAspect(StructuredProperties.class); - log.warn("Validator called with {}", structuredProperties); + log.info("Validator called with {}", structuredProperties); Map> structuredPropertiesMap = structuredProperties.getProperties().stream() .collect( @@ -412,14 +401,6 @@ private static Optional validateType( return Optional.empty(); } - private static String getValueTypeId(@Nonnull final Urn valueType) { - String valueTypeId = valueType.getId(); - if (valueTypeId.startsWith("datahub.")) { - valueTypeId = valueTypeId.split("\\.")[1]; - } - return valueTypeId; - } - private static Map> fetchPropertyAspects( @Nonnull Collection mcpItems, AspectRetriever aspectRetriever, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java b/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java index 36eab7b69e6a1..13fde9e392927 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; +import com.linkedin.common.urn.Urn; import com.linkedin.metadata.run.AspectRowSummary; import com.linkedin.metadata.run.IngestionRunSummary; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; @@ -14,6 +15,7 @@ import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.mxe.SystemMetadata; import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.util.Pair; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -227,10 +229,10 @@ public List listRuns( } @Override - public void configure() { + public void reindexAll(Collection> properties) { log.info("Setting up system metadata index"); try { - for (ReindexConfig config : buildReindexConfigs()) { + for (ReindexConfig config : buildReindexConfigs(properties)) { _indexBuilder.buildIndex(config); } } catch (IOException ie) { @@ -239,7 +241,8 @@ public void configure() { } @Override - public List buildReindexConfigs() throws IOException { + public List buildReindexConfigs( + Collection> properties) throws IOException { return List.of( _indexBuilder.buildReindexState( _indexConvention.getIndexName(INDEX_NAME), @@ -247,17 +250,6 @@ public List buildReindexConfigs() throws IOException { Collections.emptyMap())); } - @Override - public List buildReindexConfigsWithAllStructProps( - Collection properties) throws IOException { - return buildReindexConfigs(); - } - - @Override - public void reindexAll() { - configure(); - } - @VisibleForTesting @Override public void clear() { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java index 240108b191083..ce4ff53eba91b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java @@ -202,19 +202,9 @@ private static Pair toEnvAspectGener } @Override - public void configure() { - indexBuilders.reindexAll(); - } - - @Override - public List buildReindexConfigs() { - return indexBuilders.buildReindexConfigs(); - } - - @Override - public List buildReindexConfigsWithAllStructProps( - Collection properties) throws IOException { - return indexBuilders.buildReindexConfigsWithAllStructProps(properties); + public List buildReindexConfigs( + Collection> properties) throws IOException { + return indexBuilders.buildReindexConfigs(properties); } public String reindexAsync( @@ -224,8 +214,8 @@ public String reindexAsync( } @Override - public void reindexAll() { - configure(); + public void reindexAll(Collection> properties) { + indexBuilders.reindexAll(properties); } @Override @@ -308,7 +298,7 @@ public long countByFilter( .getEntityRegistry() .getEntitySpec(entityName) .getSearchableFieldTypes(), - opContext.getRetrieverContext().get().getAspectRetriever())); + opContext.getAspectRetriever())); CountRequest countRequest = new CountRequest(); countRequest.query(filterQueryBuilder); countRequest.indices(indexName); @@ -338,10 +328,7 @@ public List getAspectValues( QueryBuilders.boolQuery() .must( ESUtils.buildFilterQuery( - filter, - true, - searchableFieldTypes, - opContext.getRetrieverContext().get().getAspectRetriever())); + filter, true, searchableFieldTypes, opContext.getAspectRetriever())); filterQueryBuilder.must(QueryBuilders.matchQuery("urn", urn.toString())); // NOTE: We are interested only in the un-exploded rows as only they carry the `event` payload. filterQueryBuilder.mustNot(QueryBuilders.termQuery(MappingsBuilder.IS_EXPLODED_FIELD, true)); @@ -352,7 +339,8 @@ public List getAspectValues( .setCondition(Condition.GREATER_THAN_OR_EQUAL_TO) .setValue(startTimeMillis.toString()); filterQueryBuilder.must( - ESUtils.getQueryBuilderFromCriterion(startTimeCriterion, true, searchableFieldTypes)); + ESUtils.getQueryBuilderFromCriterion( + startTimeCriterion, true, searchableFieldTypes, opContext.getAspectRetriever())); } if (endTimeMillis != null) { Criterion endTimeCriterion = @@ -361,7 +349,8 @@ public List getAspectValues( .setCondition(Condition.LESS_THAN_OR_EQUAL_TO) .setValue(endTimeMillis.toString()); filterQueryBuilder.must( - ESUtils.getQueryBuilderFromCriterion(endTimeCriterion, true, searchableFieldTypes)); + ESUtils.getQueryBuilderFromCriterion( + endTimeCriterion, true, searchableFieldTypes, opContext.getAspectRetriever())); } final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(filterQueryBuilder); @@ -446,7 +435,7 @@ public DeleteAspectValuesResult deleteAspectValues( filter, true, opContext.getEntityRegistry().getEntitySpec(entityName).getSearchableFieldTypes(), - opContext.getRetrieverContext().get().getAspectRetriever()); + opContext.getAspectRetriever()); final Optional result = bulkProcessor @@ -482,7 +471,7 @@ public String deleteAspectValuesAsync( filter, true, opContext.getEntityRegistry().getEntitySpec(entityName).getSearchableFieldTypes(), - opContext.getRetrieverContext().get().getAspectRetriever()); + opContext.getAspectRetriever()); final int batchSize = options.getBatchSize() > 0 ? options.getBatchSize() : DEFAULT_LIMIT; TimeValue timeout = options.getTimeoutSeconds() > 0 @@ -516,7 +505,7 @@ public String reindexAsync( filter, true, opContext.getEntityRegistry().getEntitySpec(entityName).getSearchableFieldTypes(), - opContext.getRetrieverContext().get().getAspectRetriever()); + opContext.getAspectRetriever()); try { return this.reindexAsync(indexName, filterQueryBuilder, options); } catch (Exception e) { @@ -574,10 +563,7 @@ public TimeseriesScrollResult scrollAspects( QueryBuilders.boolQuery() .filter( ESUtils.buildFilterQuery( - filter, - true, - searchableFieldTypes, - opContext.getRetrieverContext().get().getAspectRetriever())); + filter, true, searchableFieldTypes, opContext.getAspectRetriever())); if (startTimeMillis != null) { Criterion startTimeCriterion = @@ -586,7 +572,8 @@ public TimeseriesScrollResult scrollAspects( .setCondition(Condition.GREATER_THAN_OR_EQUAL_TO) .setValue(startTimeMillis.toString()); filterQueryBuilder.filter( - ESUtils.getQueryBuilderFromCriterion(startTimeCriterion, true, searchableFieldTypes)); + ESUtils.getQueryBuilderFromCriterion( + startTimeCriterion, true, searchableFieldTypes, opContext.getAspectRetriever())); } if (endTimeMillis != null) { Criterion endTimeCriterion = @@ -595,7 +582,8 @@ public TimeseriesScrollResult scrollAspects( .setCondition(Condition.LESS_THAN_OR_EQUAL_TO) .setValue(endTimeMillis.toString()); filterQueryBuilder.filter( - ESUtils.getQueryBuilderFromCriterion(endTimeCriterion, true, searchableFieldTypes)); + ESUtils.getQueryBuilderFromCriterion( + endTimeCriterion, true, searchableFieldTypes, opContext.getAspectRetriever())); } SearchResponse response = diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java index b59cd3a647d71..6b67789c3e2d8 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.timeseries.elastic.indexbuilder; +import com.linkedin.common.urn.Urn; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; @@ -29,8 +30,8 @@ public class TimeseriesAspectIndexBuilders implements ElasticSearchIndexed { @Nonnull private final IndexConvention indexConvention; @Override - public void reindexAll() { - for (ReindexConfig config : buildReindexConfigs()) { + public void reindexAll(Collection> properties) { + for (ReindexConfig config : buildReindexConfigs(properties)) { try { indexBuilder.buildIndex(config); } catch (IOException e) { @@ -69,7 +70,8 @@ public String reindexAsync( } @Override - public List buildReindexConfigs() { + public List buildReindexConfigs( + Collection> properties) { return entityRegistry.getEntitySpecs().values().stream() .flatMap( entitySpec -> @@ -94,10 +96,4 @@ public List buildReindexConfigs() { }) .collect(Collectors.toList()); } - - @Override - public List buildReindexConfigsWithAllStructProps( - Collection properties) throws IOException { - return buildReindexConfigs(); - } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/query/ESAggregatedStatsDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/query/ESAggregatedStatsDAO.java index 95665450a2572..1bf96841e5fe1 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/query/ESAggregatedStatsDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/query/ESAggregatedStatsDAO.java @@ -5,6 +5,7 @@ import com.linkedin.data.schema.DataSchema; import com.linkedin.data.template.StringArray; import com.linkedin.data.template.StringArrayArray; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.TimeseriesFieldCollectionSpec; @@ -374,12 +375,13 @@ public GenericTable getAggregatedStats( filter, true, opContext.getEntityRegistry().getEntitySpec(entityName).getSearchableFieldTypes(), - opContext.getRetrieverContext().get().getAspectRetriever()); + opContext.getAspectRetriever()); AspectSpec aspectSpec = getTimeseriesAspectSpec(opContext, entityName, aspectName); // Build and attach the grouping aggregations final Pair topAndBottomAggregations = - makeGroupingAggregationBuilder(aspectSpec, null, groupingBuckets); + makeGroupingAggregationBuilder( + aspectSpec, null, groupingBuckets, opContext.getAspectRetriever()); AggregationBuilder rootAggregationBuilder = topAndBottomAggregations.getFirst(); AggregationBuilder mostNested = topAndBottomAggregations.getSecond(); @@ -462,7 +464,8 @@ private void addAggregationBuildersFromAggregationSpec( private Pair makeGroupingAggregationBuilder( AspectSpec aspectSpec, @Nullable AggregationBuilder baseAggregationBuilder, - @Nullable GroupingBucket[] groupingBuckets) { + @Nullable GroupingBucket[] groupingBuckets, + @Nonnull AspectRetriever aspectRetriever) { AggregationBuilder firstAggregationBuilder = baseAggregationBuilder; AggregationBuilder lastAggregationBuilder = baseAggregationBuilder; @@ -481,7 +484,8 @@ private Pair makeGroupingAggregationBuil } else if (curGroupingBucket.getType() == GroupingBucketType.STRING_GROUPING_BUCKET) { // Process the string grouping bucket using the 'terms' aggregation. // The field can be Keyword, Numeric, ip, boolean, or binary. - String fieldName = ESUtils.toKeywordField(curGroupingBucket.getKey(), true); + String fieldName = + ESUtils.toKeywordField(curGroupingBucket.getKey(), true, aspectRetriever); DataSchema.Type fieldType = getGroupingBucketKeyType(aspectSpec, curGroupingBucket); curAggregationBuilder = AggregationBuilders.terms(getGroupingBucketAggName(curGroupingBucket)) diff --git a/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java b/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java index 9a64e397c9184..b9a1817f476fb 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java @@ -6,6 +6,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.SetMode; import com.linkedin.dataset.UpstreamArray; import com.linkedin.dataset.UpstreamLineage; import com.linkedin.identity.CorpUserInfo; @@ -14,6 +15,7 @@ import com.linkedin.metadata.utils.PegasusUtils; import com.linkedin.mxe.SystemMetadata; import javax.annotation.Nonnull; +import javax.annotation.Nullable; public class AspectGenerationUtils { @@ -29,18 +31,28 @@ public static SystemMetadata createSystemMetadata() { return createSystemMetadata(1625792689, "run-123"); } + @Nonnull + public static SystemMetadata createSystemMetadata(long nextAspectVersion) { + return createSystemMetadata( + 1625792689, "run-123", "run-123", String.valueOf(nextAspectVersion)); + } + @Nonnull public static SystemMetadata createSystemMetadata(long lastObserved, @Nonnull String runId) { - return createSystemMetadata(lastObserved, runId, runId); + return createSystemMetadata(lastObserved, runId, runId, null); } @Nonnull public static SystemMetadata createSystemMetadata( - long lastObserved, @Nonnull String runId, @Nonnull String lastRunId) { + long lastObserved, + @Nonnull String runId, + @Nonnull String lastRunId, + @Nullable String version) { SystemMetadata metadata = new SystemMetadata(); metadata.setLastObserved(lastObserved); metadata.setRunId(runId); metadata.setLastRunId(lastRunId); + metadata.setVersion(version, SetMode.IGNORE_NULL); return metadata; } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/AspectIngestionUtils.java b/metadata-io/src/test/java/com/linkedin/metadata/AspectIngestionUtils.java index 1cffbb6e2cf21..12b12cf105196 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/AspectIngestionUtils.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/AspectIngestionUtils.java @@ -46,7 +46,7 @@ public static Map ingestCorpUserKeyAspects( .recordTemplate(aspect) .auditStamp(AspectGenerationUtils.createAuditStamp()) .systemMetadata(AspectGenerationUtils.createSystemMetadata()) - .build(opContext.getRetrieverContext().get().getAspectRetriever())); + .build(opContext.getAspectRetrieverOpt().get())); } entityService.ingestAspects( opContext, @@ -83,7 +83,7 @@ public static Map ingestCorpUserInfoAspects( .recordTemplate(aspect) .auditStamp(AspectGenerationUtils.createAuditStamp()) .systemMetadata(AspectGenerationUtils.createSystemMetadata()) - .build(opContext.getRetrieverContext().get().getAspectRetriever())); + .build(opContext.getAspectRetrieverOpt().get())); } entityService.ingestAspects( opContext, @@ -121,7 +121,7 @@ public static Map ingestChartInfoAspects( .recordTemplate(aspect) .auditStamp(AspectGenerationUtils.createAuditStamp()) .systemMetadata(AspectGenerationUtils.createSystemMetadata()) - .build(opContext.getRetrieverContext().get().getAspectRetriever())); + .build(opContext.getAspectRetrieverOpt().get())); } entityService.ingestAspects( opContext, diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java index 23513738fbc33..ef6c9e56e132b 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java @@ -92,11 +92,11 @@ private void configureComponents() { .entityRegistry(_testEntityRegistry) .build()) .graphRetriever(TestOperationContexts.emptyGraphRetriever) + .searchRetriever(TestOperationContexts.emptySearchRetriever) .build(), null, opContext -> - ((EntityServiceAspectRetriever) - opContext.getRetrieverContext().get().getAspectRetriever()) + ((EntityServiceAspectRetriever) opContext.getAspectRetrieverOpt().get()) .setSystemOperationContext(opContext)); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java index ca0c7322337ef..fe3608a2cf71d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java @@ -7,6 +7,10 @@ import com.datahub.util.RecordUtils; import com.google.common.collect.ImmutableList; import com.linkedin.common.AuditStamp; +import com.linkedin.common.FormAssociation; +import com.linkedin.common.FormAssociationArray; +import com.linkedin.common.FormVerificationAssociationArray; +import com.linkedin.common.Forms; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.container.Container; @@ -19,11 +23,17 @@ import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.graph.RelatedEntitiesResult; +import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.run.DeleteReferencesResponse; +import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.search.ScrollResult; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchEntityArray; import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.metadata.utils.SystemMetadataUtils; +import com.linkedin.mxe.MetadataChangeProposal; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.test.metadata.context.TestOperationContexts; import java.sql.Timestamp; @@ -39,17 +49,20 @@ public class DeleteEntityServiceTest { protected GraphService _graphService = Mockito.mock(GraphService.class); protected DeleteEntityService _deleteEntityService; protected UpdateIndicesService _mockUpdateIndicesService; + protected EntitySearchService _mockSearchService; public DeleteEntityServiceTest() { opContext = TestOperationContexts.systemContextNoSearchAuthorization(); _aspectDao = mock(EbeanAspectDao.class); _mockUpdateIndicesService = mock(UpdateIndicesService.class); + _mockSearchService = mock(EntitySearchService.class); PreProcessHooks preProcessHooks = new PreProcessHooks(); preProcessHooks.setUiEnabled(true); _entityServiceImpl = new EntityServiceImpl(_aspectDao, mock(EventProducer.class), true, preProcessHooks, true); _entityServiceImpl.setUpdateIndicesService(_mockUpdateIndicesService); - _deleteEntityService = new DeleteEntityService(_entityServiceImpl, _graphService); + _deleteEntityService = + new DeleteEntityService(_entityServiceImpl, _graphService, _mockSearchService); } /** @@ -119,4 +132,203 @@ public void testDeleteUniqueRefGeneratesValidMCP() { assertEquals(1, (int) response.getTotal()); assertFalse(response.getRelatedAspects().isEmpty()); } + + /** This test checks whether updating search references works properly (for forms only for now) */ + @Test + public void testDeleteSearchReferences() { + EntityService mockEntityService = Mockito.mock(EntityService.class); + DeleteEntityService deleteEntityService = + new DeleteEntityService(mockEntityService, _graphService, _mockSearchService); + + final Urn dataset = UrnUtils.toDatasetUrn("snowflake", "test", "DEV"); + final Urn form = UrnUtils.getUrn("urn:li:form:12345"); + + ScrollResult scrollResult = new ScrollResult(); + SearchEntityArray entities = new SearchEntityArray(); + SearchEntity searchEntity = new SearchEntity(); + searchEntity.setEntity(dataset); + entities.add(searchEntity); + scrollResult.setEntities(entities); + scrollResult.setNumEntities(1); + scrollResult.setScrollId("1"); + Mockito.when( + _mockSearchService.structuredScroll( + Mockito.any(OperationContext.class), + Mockito.any(), + Mockito.eq("*"), + Mockito.any(Filter.class), + Mockito.eq(null), + Mockito.eq(null), + Mockito.eq("5m"), + Mockito.eq(1000))) + .thenReturn(scrollResult); + + ScrollResult scrollResult2 = new ScrollResult(); + scrollResult2.setNumEntities(0); + Mockito.when( + _mockSearchService.structuredScroll( + Mockito.any(OperationContext.class), + Mockito.any(), + Mockito.eq("*"), + Mockito.any(Filter.class), + Mockito.eq(null), + Mockito.eq("1"), + Mockito.eq("5m"), + Mockito.eq(1000))) + .thenReturn(scrollResult2); + + Forms formsAspect = new Forms(); + FormAssociationArray incompleteForms = new FormAssociationArray(); + FormAssociation formAssociation = new FormAssociation(); + formAssociation.setUrn(form); + incompleteForms.add(formAssociation); + formsAspect.setIncompleteForms(incompleteForms); + formsAspect.setCompletedForms(new FormAssociationArray()); + formsAspect.setVerifications(new FormVerificationAssociationArray()); + Mockito.when( + mockEntityService.getLatestAspect( + Mockito.any(OperationContext.class), Mockito.eq(dataset), Mockito.eq("forms"))) + .thenReturn(formsAspect); + + // no entities with relationships on forms + final RelatedEntitiesResult mockRelatedEntities = + new RelatedEntitiesResult(0, 0, 0, ImmutableList.of()); + Mockito.when( + _graphService.findRelatedEntities( + null, + newFilter("urn", form.toString()), + null, + EMPTY_FILTER, + ImmutableList.of(), + newRelationshipFilter(EMPTY_FILTER, RelationshipDirection.INCOMING), + 0, + 10000)) + .thenReturn(mockRelatedEntities); + + final DeleteReferencesResponse response = + deleteEntityService.deleteReferencesTo(opContext, form, false); + + // ensure we ingest one MCP for cleaning up forms reference + Mockito.verify(mockEntityService, Mockito.times(1)) + .ingestProposal( + any(), + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class), + Mockito.eq(true)); + assertEquals(1, (int) response.getTotal()); + assertTrue(response.getRelatedAspects().isEmpty()); + } + + /** This test ensures we aren't issuing MCPs if there are no search references */ + @Test + public void testDeleteNoSearchReferences() { + EntityService mockEntityService = Mockito.mock(EntityService.class); + DeleteEntityService deleteEntityService = + new DeleteEntityService(mockEntityService, _graphService, _mockSearchService); + + final Urn dataset = UrnUtils.toDatasetUrn("snowflake", "test", "DEV"); + final Urn form = UrnUtils.getUrn("urn:li:form:12345"); + + ScrollResult scrollResult = new ScrollResult(); + scrollResult.setEntities(new SearchEntityArray()); + scrollResult.setNumEntities(0); + Mockito.when( + _mockSearchService.structuredScroll( + Mockito.any(OperationContext.class), + Mockito.any(), + Mockito.eq("*"), + Mockito.any(Filter.class), + Mockito.eq(null), + Mockito.eq(null), + Mockito.eq("5m"), + Mockito.eq(1000))) + .thenReturn(scrollResult); + + // no entities with relationships on forms + final RelatedEntitiesResult mockRelatedEntities = + new RelatedEntitiesResult(0, 0, 0, ImmutableList.of()); + Mockito.when( + _graphService.findRelatedEntities( + null, + newFilter("urn", form.toString()), + null, + EMPTY_FILTER, + ImmutableList.of(), + newRelationshipFilter(EMPTY_FILTER, RelationshipDirection.INCOMING), + 0, + 10000)) + .thenReturn(mockRelatedEntities); + + final DeleteReferencesResponse response = + deleteEntityService.deleteReferencesTo(opContext, form, false); + + // ensure we did not ingest anything if there are no references + Mockito.verify(mockEntityService, Mockito.times(0)) + .ingestProposal( + any(), + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class), + Mockito.eq(true)); + assertEquals(0, (int) response.getTotal()); + assertTrue(response.getRelatedAspects().isEmpty()); + } + + /** This test checks to make sure we don't issue MCPs if this is a dry-run */ + @Test + public void testDeleteSearchReferencesDryRun() { + EntityService mockEntityService = Mockito.mock(EntityService.class); + DeleteEntityService deleteEntityService = + new DeleteEntityService(mockEntityService, _graphService, _mockSearchService); + + final Urn dataset = UrnUtils.toDatasetUrn("snowflake", "test", "DEV"); + final Urn form = UrnUtils.getUrn("urn:li:form:12345"); + + ScrollResult scrollResult = new ScrollResult(); + SearchEntityArray entities = new SearchEntityArray(); + SearchEntity searchEntity = new SearchEntity(); + searchEntity.setEntity(dataset); + entities.add(searchEntity); + scrollResult.setEntities(entities); + scrollResult.setNumEntities(1); + scrollResult.setScrollId("1"); + Mockito.when( + _mockSearchService.structuredScroll( + Mockito.any(OperationContext.class), + Mockito.any(), + Mockito.eq("*"), + Mockito.any(Filter.class), + Mockito.eq(null), + Mockito.eq(null), + Mockito.eq("5m"), + Mockito.eq(1000))) + .thenReturn(scrollResult); + + // no entities with relationships on forms + final RelatedEntitiesResult mockRelatedEntities = + new RelatedEntitiesResult(0, 0, 0, ImmutableList.of()); + Mockito.when( + _graphService.findRelatedEntities( + null, + newFilter("urn", form.toString()), + null, + EMPTY_FILTER, + ImmutableList.of(), + newRelationshipFilter(EMPTY_FILTER, RelationshipDirection.INCOMING), + 0, + 10000)) + .thenReturn(mockRelatedEntities); + + final DeleteReferencesResponse response = + deleteEntityService.deleteReferencesTo(opContext, form, false); + + // ensure we do not ingest anything since this is dry-run, but the total returns 1 + Mockito.verify(mockEntityService, Mockito.times(0)) + .ingestProposal( + any(), + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class), + Mockito.eq(true)); + assertEquals(1, (int) response.getTotal()); + assertTrue(response.getRelatedAspects().isEmpty()); + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java index 76c14be5f8c13..b9f5984e57667 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java @@ -92,11 +92,11 @@ public void setupTest() { .entityRegistry(_testEntityRegistry) .build()) .graphRetriever(TestOperationContexts.emptyGraphRetriever) + .searchRetriever(TestOperationContexts.emptySearchRetriever) .build(), null, opContext -> - ((EntityServiceAspectRetriever) - opContext.getRetrieverContext().get().getAspectRetriever()) + ((EntityServiceAspectRetriever) opContext.getAspectRetrieverOpt().get()) .setSystemOperationContext(opContext)); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java index 45d4fe4f46c99..33598be8fc72b 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java @@ -160,7 +160,7 @@ public void testIngestGetEntity() throws Exception { // 2. Retrieve Entity com.linkedin.entity.Entity readEntity = - _entityServiceImpl.getEntity(opContext, entityUrn, Collections.emptySet()); + _entityServiceImpl.getEntity(opContext, entityUrn, Collections.emptySet(), true); // 3. Compare Entity Objects assertEquals( @@ -206,7 +206,7 @@ public void testAddKey() throws Exception { // 2. Retrieve Entity com.linkedin.entity.Entity readEntity = - _entityServiceImpl.getEntity(opContext, entityUrn, Collections.emptySet()); + _entityServiceImpl.getEntity(opContext, entityUrn, Collections.emptySet(), true); // 3. Compare Entity Objects assertEquals( @@ -261,7 +261,7 @@ public void testIngestGetEntities() throws Exception { // 2. Retrieve Entities Map readEntities = _entityServiceImpl.getEntities( - opContext, ImmutableSet.of(entityUrn1, entityUrn2), Collections.emptySet()); + opContext, ImmutableSet.of(entityUrn1, entityUrn2), Collections.emptySet(), true); // 3. Compare Entity Objects @@ -520,7 +520,7 @@ public void testReingestAspectsGetLatestAspects() throws Exception { String aspectName1 = AspectGenerationUtils.getAspectName(writeAspect1); pairToIngest.add(getAspectRecordPair(writeAspect1, CorpUserInfo.class)); - SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); + SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(1); _entityServiceImpl.ingestAspects( opContext, entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); @@ -586,10 +586,12 @@ public void testReingestLineageAspect() throws Exception { String aspectName1 = AspectGenerationUtils.getAspectName(upstreamLineage); pairToIngest.add(getAspectRecordPair(upstreamLineage, UpstreamLineage.class)); - SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); - _entityServiceImpl.ingestAspects( - opContext, entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); + opContext, + entityUrn, + pairToIngest, + TEST_AUDIT_STAMP, + AspectGenerationUtils.createSystemMetadata()); final MetadataChangeLog initialChangeLog = new MetadataChangeLog(); initialChangeLog.setEntityType(entityUrn.getEntityType()); @@ -601,7 +603,7 @@ public void testReingestLineageAspect() throws Exception { GenericAspect aspect = GenericRecordUtils.serializeAspect(pairToIngest.get(0).getSecond()); initialChangeLog.setAspect(aspect); - initialChangeLog.setSystemMetadata(metadata1); + initialChangeLog.setSystemMetadata(AspectGenerationUtils.createSystemMetadata(1)); initialChangeLog.setEntityKeyAspect( GenericRecordUtils.serializeAspect( EntityKeyUtils.convertUrnToEntityKey( @@ -615,9 +617,10 @@ public void testReingestLineageAspect() throws Exception { restateChangeLog.setAspectName(aspectName1); restateChangeLog.setCreated(TEST_AUDIT_STAMP); restateChangeLog.setAspect(aspect); - restateChangeLog.setSystemMetadata(metadata1); + restateChangeLog.setSystemMetadata(AspectGenerationUtils.createSystemMetadata(1)); restateChangeLog.setPreviousAspectValue(aspect); - restateChangeLog.setPreviousSystemMetadata(simulatePullFromDB(metadata1, SystemMetadata.class)); + restateChangeLog.setPreviousSystemMetadata( + simulatePullFromDB(AspectGenerationUtils.createSystemMetadata(1), SystemMetadata.class)); restateChangeLog.setEntityKeyAspect( GenericRecordUtils.serializeAspect( EntityKeyUtils.convertUrnToEntityKey( @@ -638,7 +641,11 @@ public void testReingestLineageAspect() throws Exception { clearInvocations(_mockProducer); _entityServiceImpl.ingestAspects( - opContext, entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); + opContext, + entityUrn, + pairToIngest, + TEST_AUDIT_STAMP, + AspectGenerationUtils.createSystemMetadata()); verify(_mockProducer, times(1)) .produceMetadataChangeLog( @@ -658,7 +665,7 @@ public void testReingestLineageProposal() throws Exception { final UpstreamLineage upstreamLineage = AspectGenerationUtils.createUpstreamLineage(); String aspectName1 = AspectGenerationUtils.getAspectName(upstreamLineage); - SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); + SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(1); MetadataChangeProposal mcp1 = new MetadataChangeProposal(); mcp1.setEntityType(entityUrn.getEntityType()); GenericAspect genericAspect = GenericRecordUtils.serializeAspect(upstreamLineage); @@ -687,7 +694,7 @@ public void testReingestLineageProposal() throws Exception { restateChangeLog.setAspectName(aspectName1); restateChangeLog.setCreated(TEST_AUDIT_STAMP); restateChangeLog.setAspect(genericAspect); - restateChangeLog.setSystemMetadata(metadata1); + restateChangeLog.setSystemMetadata(AspectGenerationUtils.createSystemMetadata(1)); restateChangeLog.setPreviousAspectValue(genericAspect); restateChangeLog.setPreviousSystemMetadata(simulatePullFromDB(metadata1, SystemMetadata.class)); @@ -927,28 +934,28 @@ public void testRollbackAspect() throws AssertionError { .recordTemplate(writeAspect1) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), ChangeItemImpl.builder() .urn(entityUrn2) .aspectName(aspectName) .recordTemplate(writeAspect2) .auditStamp(TEST_AUDIT_STAMP) .systemMetadata(metadata1) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), ChangeItemImpl.builder() .urn(entityUrn3) .aspectName(aspectName) .recordTemplate(writeAspect3) .auditStamp(TEST_AUDIT_STAMP) .systemMetadata(metadata1) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), ChangeItemImpl.builder() .urn(entityUrn1) .aspectName(aspectName) .recordTemplate(writeAspect1Overwrite) .systemMetadata(metadata2) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever())); + .build(opContext.getAspectRetrieverOpt().get())); _entityServiceImpl.ingestAspects( opContext, AspectsBatchImpl.builder() @@ -1019,21 +1026,21 @@ public void testRollbackKey() throws AssertionError { .recordTemplate(writeAspect1) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), ChangeItemImpl.builder() .urn(entityUrn1) .aspectName(keyAspectName) .recordTemplate(writeKey1) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), ChangeItemImpl.builder() .urn(entityUrn1) .aspectName(aspectName) .recordTemplate(writeAspect1Overwrite) .systemMetadata(metadata2) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever())); + .build(opContext.getAspectRetrieverOpt().get())); _entityServiceImpl.ingestAspects( opContext, AspectsBatchImpl.builder() @@ -1112,35 +1119,35 @@ public void testRollbackUrn() throws AssertionError { .recordTemplate(writeAspect1) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), ChangeItemImpl.builder() .urn(entityUrn1) .aspectName(keyAspectName) .recordTemplate(writeKey1) .auditStamp(TEST_AUDIT_STAMP) .systemMetadata(metadata1) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), ChangeItemImpl.builder() .urn(entityUrn2) .aspectName(aspectName) .recordTemplate(writeAspect2) .auditStamp(TEST_AUDIT_STAMP) .systemMetadata(metadata1) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), ChangeItemImpl.builder() .urn(entityUrn3) .aspectName(aspectName) .recordTemplate(writeAspect3) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), ChangeItemImpl.builder() .urn(entityUrn1) .aspectName(aspectName) .recordTemplate(writeAspect1Overwrite) .systemMetadata(metadata2) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever())); + .build(opContext.getAspectRetrieverOpt().get())); _entityServiceImpl.ingestAspects( opContext, AspectsBatchImpl.builder() @@ -1177,8 +1184,10 @@ public void testIngestGetLatestAspect() throws AssertionError { CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); String aspectName = AspectGenerationUtils.getAspectName(writeAspect1); - SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(1625792689, "run-123"); - SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-456"); + SystemMetadata metadata1 = + AspectGenerationUtils.createSystemMetadata(1625792689, "run-123", "run-123", "1"); + SystemMetadata metadata2 = + AspectGenerationUtils.createSystemMetadata(1635792689, "run-456", "run-456", "2"); List items = List.of( @@ -1188,7 +1197,7 @@ public void testIngestGetLatestAspect() throws AssertionError { .recordTemplate(writeAspect1) .auditStamp(TEST_AUDIT_STAMP) .systemMetadata(metadata1) - .build(opContext.getRetrieverContext().get().getAspectRetriever())); + .build(opContext.getAspectRetrieverOpt().get())); _entityServiceImpl.ingestAspects( opContext, AspectsBatchImpl.builder() @@ -1244,7 +1253,7 @@ public void testIngestGetLatestAspect() throws AssertionError { .recordTemplate(writeAspect2) .auditStamp(TEST_AUDIT_STAMP) .systemMetadata(metadata2) - .build(opContext.getRetrieverContext().get().getAspectRetriever())); + .build(opContext.getAspectRetrieverOpt().get())); _entityServiceImpl.ingestAspects( opContext, AspectsBatchImpl.builder() @@ -1287,8 +1296,10 @@ public void testIngestGetLatestEnvelopedAspect() throws Exception { CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); String aspectName = AspectGenerationUtils.getAspectName(writeAspect1); - SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(1625792689, "run-123"); - SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-456"); + SystemMetadata metadata1 = + AspectGenerationUtils.createSystemMetadata(1625792689, "run-123", "run-123", "1"); + SystemMetadata metadata2 = + AspectGenerationUtils.createSystemMetadata(1635792689, "run-456", "run-456", "2"); List items = List.of( @@ -1298,7 +1309,7 @@ public void testIngestGetLatestEnvelopedAspect() throws Exception { .recordTemplate(writeAspect1) .auditStamp(TEST_AUDIT_STAMP) .systemMetadata(metadata1) - .build(opContext.getRetrieverContext().get().getAspectRetriever())); + .build(opContext.getAspectRetrieverOpt().get())); _entityServiceImpl.ingestAspects( opContext, AspectsBatchImpl.builder() @@ -1325,7 +1336,7 @@ public void testIngestGetLatestEnvelopedAspect() throws Exception { .recordTemplate(writeAspect2) .systemMetadata(metadata2) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever())); + .build(opContext.getAspectRetrieverOpt().get())); _entityServiceImpl.ingestAspects( opContext, AspectsBatchImpl.builder() @@ -1384,7 +1395,7 @@ public void testIngestSameAspect() throws AssertionError { SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(1625792689, "run-123"); SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-456"); SystemMetadata metadata3 = - AspectGenerationUtils.createSystemMetadata(1635792689, "run-123", "run-456"); + AspectGenerationUtils.createSystemMetadata(1635792689, "run-123", "run-456", "1"); List items = List.of( @@ -1394,7 +1405,7 @@ public void testIngestSameAspect() throws AssertionError { .recordTemplate(writeAspect1) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever())); + .build(opContext.getAspectRetrieverOpt().get())); _entityServiceImpl.ingestAspects( opContext, AspectsBatchImpl.builder() @@ -1450,7 +1461,7 @@ public void testIngestSameAspect() throws AssertionError { .recordTemplate(writeAspect2) .systemMetadata(metadata2) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever())); + .build(opContext.getAspectRetrieverOpt().get())); _entityServiceImpl.ingestAspects( opContext, AspectsBatchImpl.builder() @@ -1488,8 +1499,6 @@ public void testIngestSameAspect() throws AssertionError { public void testRetention() throws AssertionError { Urn entityUrn = UrnUtils.getUrn("urn:li:corpuser:test1"); - SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); - String aspectName = AspectGenerationUtils.getAspectName(new CorpUserInfo()); // Ingest CorpUserInfo Aspect @@ -1509,44 +1518,44 @@ public void testRetention() throws AssertionError { .urn(entityUrn) .aspectName(aspectName) .recordTemplate(writeAspect1) - .systemMetadata(metadata1) + .systemMetadata(AspectGenerationUtils.createSystemMetadata()) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), ChangeItemImpl.builder() .urn(entityUrn) .aspectName(aspectName) .recordTemplate(writeAspect1a) - .systemMetadata(metadata1) + .systemMetadata(AspectGenerationUtils.createSystemMetadata()) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), ChangeItemImpl.builder() .urn(entityUrn) .aspectName(aspectName) .recordTemplate(writeAspect1b) - .systemMetadata(metadata1) + .systemMetadata(AspectGenerationUtils.createSystemMetadata()) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), ChangeItemImpl.builder() .urn(entityUrn) .aspectName(aspectName2) .recordTemplate(writeAspect2) - .systemMetadata(metadata1) + .systemMetadata(AspectGenerationUtils.createSystemMetadata()) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), ChangeItemImpl.builder() .urn(entityUrn) .aspectName(aspectName2) .recordTemplate(writeAspect2a) - .systemMetadata(metadata1) + .systemMetadata(AspectGenerationUtils.createSystemMetadata()) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), ChangeItemImpl.builder() .urn(entityUrn) .aspectName(aspectName2) .recordTemplate(writeAspect2b) - .systemMetadata(metadata1) + .systemMetadata(AspectGenerationUtils.createSystemMetadata()) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever())); + .build(opContext.getAspectRetrieverOpt().get())); _entityServiceImpl.ingestAspects( opContext, AspectsBatchImpl.builder() @@ -1585,16 +1594,16 @@ public void testRetention() throws AssertionError { .urn(entityUrn) .aspectName(aspectName) .recordTemplate(writeAspect1c) - .systemMetadata(metadata1) + .systemMetadata(AspectGenerationUtils.createSystemMetadata()) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever()), + .build(opContext.getAspectRetrieverOpt().get()), ChangeItemImpl.builder() .urn(entityUrn) .aspectName(aspectName2) .recordTemplate(writeAspect2c) - .systemMetadata(metadata1) + .systemMetadata(AspectGenerationUtils.createSystemMetadata()) .auditStamp(TEST_AUDIT_STAMP) - .build(opContext.getRetrieverContext().get().getAspectRetriever())); + .build(opContext.getAspectRetrieverOpt().get())); _entityServiceImpl.ingestAspects( opContext, AspectsBatchImpl.builder() @@ -1634,32 +1643,60 @@ public void testRetention() throws AssertionError { public void testIngestAspectIfNotPresent() throws AssertionError { Urn entityUrn = UrnUtils.getUrn("urn:li:corpuser:test1"); - SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); - String aspectName = AspectGenerationUtils.getAspectName(new CorpUserInfo()); // Ingest CorpUserInfo Aspect CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); _entityServiceImpl.ingestAspectIfNotPresent( - opContext, entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); + opContext, + entityUrn, + aspectName, + writeAspect1, + TEST_AUDIT_STAMP, + AspectGenerationUtils.createSystemMetadata()); CorpUserInfo writeAspect1a = AspectGenerationUtils.createCorpUserInfo("email_a@test.com"); _entityServiceImpl.ingestAspectIfNotPresent( - opContext, entityUrn, aspectName, writeAspect1a, TEST_AUDIT_STAMP, metadata1); + opContext, + entityUrn, + aspectName, + writeAspect1a, + TEST_AUDIT_STAMP, + AspectGenerationUtils.createSystemMetadata()); CorpUserInfo writeAspect1b = AspectGenerationUtils.createCorpUserInfo("email_b@test.com"); _entityServiceImpl.ingestAspectIfNotPresent( - opContext, entityUrn, aspectName, writeAspect1b, TEST_AUDIT_STAMP, metadata1); + opContext, + entityUrn, + aspectName, + writeAspect1b, + TEST_AUDIT_STAMP, + AspectGenerationUtils.createSystemMetadata()); String aspectName2 = AspectGenerationUtils.getAspectName(new Status()); // Ingest Status Aspect Status writeAspect2 = new Status().setRemoved(true); _entityServiceImpl.ingestAspectIfNotPresent( - opContext, entityUrn, aspectName2, writeAspect2, TEST_AUDIT_STAMP, metadata1); + opContext, + entityUrn, + aspectName2, + writeAspect2, + TEST_AUDIT_STAMP, + AspectGenerationUtils.createSystemMetadata()); Status writeAspect2a = new Status().setRemoved(false); _entityServiceImpl.ingestAspectIfNotPresent( - opContext, entityUrn, aspectName2, writeAspect2a, TEST_AUDIT_STAMP, metadata1); + opContext, + entityUrn, + aspectName2, + writeAspect2a, + TEST_AUDIT_STAMP, + AspectGenerationUtils.createSystemMetadata()); Status writeAspect2b = new Status().setRemoved(true); _entityServiceImpl.ingestAspectIfNotPresent( - opContext, entityUrn, aspectName2, writeAspect2b, TEST_AUDIT_STAMP, metadata1); + opContext, + entityUrn, + aspectName2, + writeAspect2b, + TEST_AUDIT_STAMP, + AspectGenerationUtils.createSystemMetadata()); assertEquals(_entityServiceImpl.getAspect(opContext, entityUrn, aspectName, 0), writeAspect1); assertEquals(_entityServiceImpl.getAspect(opContext, entityUrn, aspectName2, 0), writeAspect2); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java index c151c1f381ce9..1aebc48153bbe 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java @@ -2163,7 +2163,7 @@ public void testHighlyConnectedGraphWalk() throws Exception { } assertEquals(new HashSet<>(relatedEntities.getEntities()), expectedRelatedEntities); - Urn root = UrnUtils.getUrn(relatedEntities.getEntities().get(0).getUrn()); + Urn root = dataset1Urn; EntityLineageResult lineageResult = getGraphService(false) .getLineage( @@ -2180,13 +2180,18 @@ public void testHighlyConnectedGraphWalk() throws Exception { 1000, 100, new LineageFlags().setEntitiesExploredPerHopLimit(5)); - assertEquals(lineageResult.getRelationships().size(), 19); + // Unable to explore all paths because multi is disabled, but will be at least 5 since it will + // explore 5 edges + assertTrue( + lineageResult.getRelationships().size() >= 5 + && lineageResult.getRelationships().size() < 20, + "Size was: " + lineageResult.getRelationships().size()); LineageRelationshipArray relationships = lineageResult.getRelationships(); int maxDegree = relationships.stream() .flatMap(relationship -> relationship.getDegrees().stream()) .reduce(0, Math::max); - assertEquals(maxDegree, 1); + assertTrue(maxDegree > 1); EntityLineageResult lineageResultMulti = getGraphService(true) @@ -2205,13 +2210,16 @@ public void testHighlyConnectedGraphWalk() throws Exception { 100, new LineageFlags().setEntitiesExploredPerHopLimit(5)); - assertEquals(lineageResultMulti.getRelationships().size(), 20); + assertTrue( + lineageResultMulti.getRelationships().size() >= 5 + && lineageResultMulti.getRelationships().size() <= 20, + "Size was: " + lineageResultMulti.getRelationships().size()); relationships = lineageResultMulti.getRelationships(); maxDegree = relationships.stream() .flatMap(relationship -> relationship.getDegrees().stream()) .reduce(0, Math::max); - assertTrue(maxDegree > 4); + assertTrue(maxDegree >= 2); // Reset graph service getGraphService(); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/dgraph/DgraphContainer.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/dgraph/DgraphContainer.java index 481db53eafbbe..9c67c610196ed 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/dgraph/DgraphContainer.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/dgraph/DgraphContainer.java @@ -224,9 +224,6 @@ public Set getLivenessCheckPortNumbers() { return Stream.of(getHttpPort(), getGrpcPort()).map(this::getMappedPort).collect(toSet()); } - @Override - protected void configure() {} - public int getHttpPort() { return getMappedPort(HTTP_PORT); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java index 23b4c82ca0566..b4ad5ce61d8f4 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java @@ -72,7 +72,7 @@ public abstract class SearchGraphServiceTestBase extends GraphServiceTestBase { @BeforeClass public void setup() { _client = buildService(_enableMultiPathSearch); - _client.configure(); + _client.reindexAll(Collections.emptySet()); } @BeforeMethod @@ -121,7 +121,7 @@ protected GraphService getGraphService(boolean enableMultiPathSearch) { if (enableMultiPathSearch != _enableMultiPathSearch) { _enableMultiPathSearch = enableMultiPathSearch; _client = buildService(enableMultiPathSearch); - _client.configure(); + _client.reindexAll(Collections.emptySet()); } return _client; } @@ -430,7 +430,10 @@ public void testExplored() throws Exception { Assert.assertTrue(Boolean.TRUE.equals(result.getRelationships().get(0).isExplored())); EntityLineageResult result2 = getUpstreamLineage(dataset2Urn, null, null, 10, 0); - Assert.assertTrue(result2.getRelationships().get(0).isExplored() == null); + Assert.assertTrue(result2.getRelationships().isEmpty()); + + EntityLineageResult result3 = getUpstreamLineage(dataset2Urn, null, null, 10, 1); + Assert.assertTrue(result3.getRelationships().get(0).isExplored()); } /** diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java index 3f03c64c6f921..3dbbfb2cebc3f 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java @@ -126,7 +126,7 @@ public void setup() throws RemoteInvocationException, URISyntaxException { .asSession(RequestContext.TEST, Authorizer.EMPTY, TestOperationContexts.TEST_USER_AUTH); settingsBuilder = new SettingsBuilder(null); elasticSearchService = buildEntitySearchService(); - elasticSearchService.configure(); + elasticSearchService.reindexAll(Collections.emptySet()); cacheManager = new ConcurrentMapCacheManager(); graphService = mock(GraphService.class); resetService(true, false); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java index d5a21cfecee27..a610cf95f827a 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTestBase.java @@ -39,6 +39,7 @@ import io.datahubproject.metadata.context.RequestContext; import io.datahubproject.test.metadata.context.TestOperationContexts; import java.net.URISyntaxException; +import java.util.Collections; import javax.annotation.Nonnull; import org.opensearch.client.RestHighLevelClient; import org.springframework.cache.CacheManager; @@ -83,7 +84,7 @@ public void setup() throws RemoteInvocationException, URISyntaxException { settingsBuilder = new SettingsBuilder(null); elasticSearchService = buildEntitySearchService(); - elasticSearchService.configure(); + elasticSearchService.reindexAll(Collections.emptySet()); cacheManager = new ConcurrentMapCacheManager(); resetSearchService(); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java index c3dcf3aaee9b7..58574025aeeac 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/TestEntityTestBase.java @@ -25,6 +25,7 @@ import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.test.metadata.context.TestOperationContexts; +import java.util.Collections; import java.util.List; import javax.annotation.Nonnull; import org.opensearch.client.RestHighLevelClient; @@ -64,7 +65,7 @@ public void setup() { new SnapshotEntityRegistry(new Snapshot()), new IndexConventionImpl("es_service_test")); settingsBuilder = new SettingsBuilder(null); elasticSearchService = buildService(); - elasticSearchService.configure(); + elasticSearchService.reindexAll(Collections.emptySet()); } @BeforeMethod diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/IndexBuilderTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/IndexBuilderTestBase.java index 0858c3dd7eb99..92ca4c5ed8a05 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/IndexBuilderTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/IndexBuilderTestBase.java @@ -57,6 +57,7 @@ public void setup() { Map.of(), false, false, + false, new ElasticSearchConfiguration(), gitVersion); } @@ -103,6 +104,7 @@ public void testESIndexBuilderCreation() throws Exception { Map.of(), false, false, + false, new ElasticSearchConfiguration(), gitVersion); customIndexBuilder.buildIndex(TEST_INDEX_NAME, Map.of(), Map.of()); @@ -126,6 +128,7 @@ public void testMappingReindex() throws Exception { Map.of(), false, true, + false, new ElasticSearchConfiguration(), gitVersion); @@ -197,6 +200,7 @@ public void testSettingsNumberOfShardsReindex() throws Exception { Map.of(), true, false, + false, new ElasticSearchConfiguration(), gitVersion); @@ -238,6 +242,7 @@ public void testSettingsNoReindex() throws Exception { Map.of(), true, false, + false, new ElasticSearchConfiguration(), gitVersion), new ESIndexBuilder( @@ -249,6 +254,7 @@ public void testSettingsNoReindex() throws Exception { Map.of(), true, false, + false, new ElasticSearchConfiguration(), gitVersion), new ESIndexBuilder( @@ -260,6 +266,7 @@ public void testSettingsNoReindex() throws Exception { Map.of(), false, false, + false, new ElasticSearchConfiguration(), gitVersion), new ESIndexBuilder( @@ -271,6 +278,7 @@ public void testSettingsNoReindex() throws Exception { Map.of(), false, false, + false, new ElasticSearchConfiguration(), gitVersion)); @@ -314,6 +322,7 @@ public void testCopyStructuredPropertyMappings() throws Exception { Map.of(), false, true, + false, new ElasticSearchConfiguration(), gitVersion); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java index 9185e2e7ee072..75da2bc62aaad 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java @@ -7,6 +7,8 @@ import com.google.common.collect.ImmutableMap; import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; import com.linkedin.metadata.TestEntitySpecBuilder; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.EntitySpecBuilder; @@ -15,6 +17,7 @@ import com.linkedin.metadata.search.elasticsearch.indexbuilder.MappingsBuilder; import com.linkedin.metadata.search.elasticsearch.query.request.TestSearchFieldConfig; import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.util.Pair; import java.io.Serializable; import java.net.URISyntaxException; import java.util.List; @@ -178,19 +181,25 @@ public void testGetMappingsWithStructuredProperty() throws URISyntaxException { // Test that a structured property that does not apply to the entity does not alter the mappings StructuredPropertyDefinition structPropNotForThisEntity = new StructuredPropertyDefinition() + .setVersion(null, SetMode.REMOVE_IF_NULL) .setQualifiedName("propNotForThis") .setDisplayName("propNotForThis") .setEntityTypes(new UrnArray(Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "dataset"))) .setValueType(Urn.createFromString("urn:li:logicalType:STRING")); Map resultWithOnlyUnrelatedStructuredProp = MappingsBuilder.getMappings( - TestEntitySpecBuilder.getSpec(), List.of(structPropNotForThisEntity)); + TestEntitySpecBuilder.getSpec(), + List.of( + Pair.of( + UrnUtils.getUrn("urn:li:structuredProperty:propNotForThis"), + structPropNotForThisEntity))); assertEquals(resultWithOnlyUnrelatedStructuredProp, resultWithoutStructuredProps); // Test that a structured property that does apply to this entity is included in the mappings String fqnOfRelatedProp = "propForThis"; StructuredPropertyDefinition structPropForThisEntity = new StructuredPropertyDefinition() + .setVersion(null, SetMode.REMOVE_IF_NULL) .setQualifiedName(fqnOfRelatedProp) .setDisplayName("propForThis") .setEntityTypes( @@ -200,7 +209,11 @@ public void testGetMappingsWithStructuredProperty() throws URISyntaxException { .setValueType(Urn.createFromString("urn:li:logicalType:STRING")); Map resultWithOnlyRelatedStructuredProp = MappingsBuilder.getMappings( - TestEntitySpecBuilder.getSpec(), List.of(structPropForThisEntity)); + TestEntitySpecBuilder.getSpec(), + List.of( + Pair.of( + UrnUtils.getUrn("urn:li:structuredProperty:propForThis"), + structPropForThisEntity))); assertNotEquals(resultWithOnlyRelatedStructuredProp, resultWithoutStructuredProps); Map fieldsBefore = (Map) resultWithoutStructuredProps.get("properties"); @@ -231,7 +244,95 @@ public void testGetMappingsWithStructuredProperty() throws URISyntaxException { Map resultWithBothStructuredProps = MappingsBuilder.getMappings( TestEntitySpecBuilder.getSpec(), - List.of(structPropForThisEntity, structPropNotForThisEntity)); + List.of( + Pair.of( + UrnUtils.getUrn("urn:li:structuredProperty:propForThis"), + structPropForThisEntity), + Pair.of( + UrnUtils.getUrn("urn:li:structuredProperty:propNotForThis"), + structPropNotForThisEntity))); + assertEquals(resultWithBothStructuredProps, resultWithOnlyRelatedStructuredProp); + } + + @Test + public void testGetMappingsWithStructuredPropertyV1() throws URISyntaxException { + // Baseline comparison: Mappings with no structured props + Map resultWithoutStructuredProps = + MappingsBuilder.getMappings(TestEntitySpecBuilder.getSpec()); + + // Test that a structured property that does not apply to the entity does not alter the mappings + StructuredPropertyDefinition structPropNotForThisEntity = + new StructuredPropertyDefinition() + .setVersion("00000000000001") + .setQualifiedName("propNotForThis") + .setDisplayName("propNotForThis") + .setEntityTypes(new UrnArray(Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "dataset"))) + .setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + Map resultWithOnlyUnrelatedStructuredProp = + MappingsBuilder.getMappings( + TestEntitySpecBuilder.getSpec(), + List.of( + Pair.of( + UrnUtils.getUrn("urn:li:structuredProperty:propNotForThis"), + structPropNotForThisEntity))); + assertEquals(resultWithOnlyUnrelatedStructuredProp, resultWithoutStructuredProps); + + // Test that a structured property that does apply to this entity is included in the mappings + String fqnOfRelatedProp = "propForThis"; + StructuredPropertyDefinition structPropForThisEntity = + new StructuredPropertyDefinition() + .setVersion("00000000000001") + .setQualifiedName(fqnOfRelatedProp) + .setDisplayName("propForThis") + .setEntityTypes( + new UrnArray( + Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "dataset"), + Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "testEntity"))) + .setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + Map resultWithOnlyRelatedStructuredProp = + MappingsBuilder.getMappings( + TestEntitySpecBuilder.getSpec(), + List.of( + Pair.of( + UrnUtils.getUrn("urn:li:structuredProperty:propForThis"), + structPropForThisEntity))); + assertNotEquals(resultWithOnlyRelatedStructuredProp, resultWithoutStructuredProps); + Map fieldsBefore = + (Map) resultWithoutStructuredProps.get("properties"); + Map fieldsAfter = + (Map) resultWithOnlyRelatedStructuredProp.get("properties"); + assertEquals(fieldsAfter.size(), fieldsBefore.size() + 1); + + Map structProps = (Map) fieldsAfter.get("structuredProperties"); + fieldsAfter = (Map) structProps.get("properties"); + + String newField = + fieldsAfter.keySet().stream() + .filter(field -> !fieldsBefore.containsKey(field)) + .findFirst() + .get(); + assertEquals(newField, "_versioned." + fqnOfRelatedProp + ".00000000000001.string"); + assertEquals( + fieldsAfter.get(newField), + Map.of( + "normalizer", + "keyword_normalizer", + "type", + "keyword", + "fields", + Map.of("keyword", Map.of("type", "keyword")))); + + // Test that only structured properties that apply are included + Map resultWithBothStructuredProps = + MappingsBuilder.getMappings( + TestEntitySpecBuilder.getSpec(), + List.of( + Pair.of( + UrnUtils.getUrn("urn:li:structuredProperty:propForThis"), + structPropForThisEntity), + Pair.of( + UrnUtils.getUrn("urn:li:structuredProperty:propNotForThis"), + structPropNotForThisEntity))); assertEquals(resultWithBothStructuredProps, resultWithOnlyRelatedStructuredProp); } @@ -239,6 +340,7 @@ public void testGetMappingsWithStructuredProperty() throws URISyntaxException { public void testGetMappingsForStructuredProperty() throws URISyntaxException { StructuredPropertyDefinition testStructProp = new StructuredPropertyDefinition() + .setVersion(null, SetMode.REMOVE_IF_NULL) .setQualifiedName("testProp") .setDisplayName("exampleProp") .setEntityTypes( @@ -247,10 +349,14 @@ public void testGetMappingsForStructuredProperty() throws URISyntaxException { Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "testEntity"))) .setValueType(Urn.createFromString("urn:li:logicalType:STRING")); Map structuredPropertyFieldMappings = - MappingsBuilder.getMappingsForStructuredProperty(List.of(testStructProp)); + MappingsBuilder.getMappingsForStructuredProperty( + List.of( + (Pair.of( + UrnUtils.getUrn("urn:li:structuredProperties:testProp"), testStructProp)))); assertEquals(structuredPropertyFieldMappings.size(), 1); String keyInMap = structuredPropertyFieldMappings.keySet().stream().findFirst().get(); assertEquals(keyInMap, "testProp"); + Object mappings = structuredPropertyFieldMappings.get(keyInMap); assertEquals( mappings, @@ -264,6 +370,7 @@ public void testGetMappingsForStructuredProperty() throws URISyntaxException { StructuredPropertyDefinition propWithNumericType = new StructuredPropertyDefinition() + .setVersion(null, SetMode.REMOVE_IF_NULL) .setQualifiedName("testPropNumber") .setDisplayName("examplePropNumber") .setEntityTypes( @@ -272,7 +379,11 @@ public void testGetMappingsForStructuredProperty() throws URISyntaxException { Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "testEntity"))) .setValueType(Urn.createFromString("urn:li:logicalType:NUMBER")); Map structuredPropertyFieldMappingsNumber = - MappingsBuilder.getMappingsForStructuredProperty(List.of(propWithNumericType)); + MappingsBuilder.getMappingsForStructuredProperty( + List.of( + Pair.of( + UrnUtils.getUrn("urn:li:structuredProperties:testPropNumber"), + propWithNumericType))); assertEquals(structuredPropertyFieldMappingsNumber.size(), 1); keyInMap = structuredPropertyFieldMappingsNumber.keySet().stream().findFirst().get(); assertEquals("testPropNumber", keyInMap); @@ -280,6 +391,61 @@ public void testGetMappingsForStructuredProperty() throws URISyntaxException { assertEquals(Map.of("type", "double"), mappings); } + @Test + public void testGetMappingsForStructuredPropertyV1() throws URISyntaxException { + StructuredPropertyDefinition testStructProp = + new StructuredPropertyDefinition() + .setVersion("00000000000001") + .setQualifiedName("testProp") + .setDisplayName("exampleProp") + .setEntityTypes( + new UrnArray( + Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "dataset"), + Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "testEntity"))) + .setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + Map structuredPropertyFieldMappings = + MappingsBuilder.getMappingsForStructuredProperty( + List.of( + (Pair.of( + UrnUtils.getUrn("urn:li:structuredProperties:testProp"), testStructProp)))); + assertEquals(structuredPropertyFieldMappings.size(), 1); + String keyInMap = structuredPropertyFieldMappings.keySet().stream().findFirst().get(); + assertEquals(keyInMap, "_versioned.testProp.00000000000001.string"); + + Object mappings = structuredPropertyFieldMappings.get(keyInMap); + assertEquals( + mappings, + Map.of( + "type", + "keyword", + "normalizer", + "keyword_normalizer", + "fields", + Map.of("keyword", Map.of("type", "keyword")))); + + StructuredPropertyDefinition propWithNumericType = + new StructuredPropertyDefinition() + .setVersion("00000000000001") + .setQualifiedName("testPropNumber") + .setDisplayName("examplePropNumber") + .setEntityTypes( + new UrnArray( + Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "dataset"), + Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "testEntity"))) + .setValueType(Urn.createFromString("urn:li:logicalType:NUMBER")); + Map structuredPropertyFieldMappingsNumber = + MappingsBuilder.getMappingsForStructuredProperty( + List.of( + Pair.of( + UrnUtils.getUrn("urn:li:structuredProperty:testPropNumber"), + propWithNumericType))); + assertEquals(structuredPropertyFieldMappingsNumber.size(), 1); + keyInMap = structuredPropertyFieldMappingsNumber.keySet().stream().findFirst().get(); + assertEquals(keyInMap, "_versioned.testPropNumber.00000000000001.number"); + mappings = structuredPropertyFieldMappingsNumber.get(keyInMap); + assertEquals(Map.of("type", "double"), mappings); + } + @Test public void testRefMappingsBuilder() { EntityRegistry entityRegistry = getTestEntityRegistry(); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java index 43ae6bd7a48b7..0ea2340ae8217 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java @@ -1,21 +1,31 @@ package com.linkedin.metadata.search.query.request; +import static com.linkedin.metadata.Constants.DATA_TYPE_URN_PREFIX; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; import static com.linkedin.metadata.utils.SearchUtil.*; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.SetMode; +import com.linkedin.entity.Aspect; import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.annotation.SearchableAnnotation; import com.linkedin.metadata.search.elasticsearch.query.request.AggregationQueryBuilder; import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.structured.StructuredPropertyDefinition; import io.datahubproject.test.metadata.context.TestOperationContexts; import java.net.URISyntaxException; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -28,10 +38,73 @@ public class AggregationQueryBuilderTest { private static AspectRetriever aspectRetriever; + private static AspectRetriever aspectRetrieverV1; @BeforeClass public static void setup() throws RemoteInvocationException, URISyntaxException { - aspectRetriever = TestOperationContexts.emptyAspectRetriever(null); + Urn helloUrn = Urn.createFromString("urn:li:structuredProperty:hello"); + Urn abFghTenUrn = Urn.createFromString("urn:li:structuredProperty:ab.fgh.ten"); + + // legacy + aspectRetriever = mock(AspectRetriever.class); + when(aspectRetriever.getEntityRegistry()) + .thenReturn(TestOperationContexts.defaultEntityRegistry()); + + StructuredPropertyDefinition structPropHelloDefinition = new StructuredPropertyDefinition(); + structPropHelloDefinition.setVersion(null, SetMode.REMOVE_IF_NULL); + structPropHelloDefinition.setValueType(Urn.createFromString(DATA_TYPE_URN_PREFIX + "string")); + structPropHelloDefinition.setQualifiedName("hello"); + when(aspectRetriever.getLatestAspectObjects(eq(Set.of(helloUrn)), anySet())) + .thenReturn( + Map.of( + helloUrn, + Map.of( + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, + new Aspect(structPropHelloDefinition.data())))); + + StructuredPropertyDefinition structPropAbFghTenDefinition = new StructuredPropertyDefinition(); + structPropAbFghTenDefinition.setVersion(null, SetMode.REMOVE_IF_NULL); + structPropAbFghTenDefinition.setValueType( + Urn.createFromString(DATA_TYPE_URN_PREFIX + "string")); + structPropAbFghTenDefinition.setQualifiedName("ab.fgh.ten"); + when(aspectRetriever.getLatestAspectObjects(eq(Set.of(abFghTenUrn)), anySet())) + .thenReturn( + Map.of( + abFghTenUrn, + Map.of( + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, + new Aspect(structPropAbFghTenDefinition.data())))); + + // V1 + aspectRetrieverV1 = mock(AspectRetriever.class); + when(aspectRetrieverV1.getEntityRegistry()) + .thenReturn(TestOperationContexts.defaultEntityRegistry()); + + StructuredPropertyDefinition structPropHelloDefinitionV1 = new StructuredPropertyDefinition(); + structPropHelloDefinitionV1.setVersion("00000000000001"); + structPropHelloDefinitionV1.setValueType(Urn.createFromString(DATA_TYPE_URN_PREFIX + "string")); + structPropHelloDefinitionV1.setQualifiedName("hello"); + when(aspectRetrieverV1.getLatestAspectObjects(eq(Set.of(helloUrn)), anySet())) + .thenReturn( + Map.of( + helloUrn, + Map.of( + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, + new Aspect(structPropHelloDefinitionV1.data())))); + + StructuredPropertyDefinition structPropAbFghTenDefinitionV1 = + new StructuredPropertyDefinition(); + structPropAbFghTenDefinitionV1.setVersion("00000000000001"); + structPropAbFghTenDefinitionV1.setValueType( + Urn.createFromString(DATA_TYPE_URN_PREFIX + "string")); + structPropAbFghTenDefinitionV1.setQualifiedName("ab.fgh.ten"); + when(aspectRetrieverV1.getLatestAspectObjects(eq(Set.of(abFghTenUrn)), anySet())) + .thenReturn( + Map.of( + abFghTenUrn, + Map.of( + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, + new Aspect(structPropAbFghTenDefinitionV1.data())))); } @Test @@ -171,26 +244,69 @@ public void testAggregateOverStructuredProperty() { List aggs = builder.getAggregations( - TestOperationContexts.systemContextNoSearchAuthorization(), + TestOperationContexts.systemContextNoSearchAuthorization(aspectRetriever), List.of("structuredProperties.ab.fgh.ten")); Assert.assertEquals(aggs.size(), 1); AggregationBuilder aggBuilder = aggs.get(0); Assert.assertTrue(aggBuilder instanceof TermsAggregationBuilder); TermsAggregationBuilder agg = (TermsAggregationBuilder) aggBuilder; // Check that field name is sanitized to correct field name - Assert.assertEquals(agg.field(), "structuredProperties.ab_fgh_ten"); + Assert.assertEquals( + agg.field(), + "structuredProperties.ab_fgh_ten.keyword", + "Terms aggregate must be on a keyword or subfield keyword"); // Two structured properties aggs = builder.getAggregations( - TestOperationContexts.systemContextNoSearchAuthorization(), + TestOperationContexts.systemContextNoSearchAuthorization(aspectRetriever), List.of("structuredProperties.ab.fgh.ten", "structuredProperties.hello")); Assert.assertEquals(aggs.size(), 2); Assert.assertEquals( aggs.stream() .map(aggr -> ((TermsAggregationBuilder) aggr).field()) .collect(Collectors.toSet()), - Set.of("structuredProperties.ab_fgh_ten", "structuredProperties.hello")); + Set.of("structuredProperties.ab_fgh_ten.keyword", "structuredProperties.hello.keyword")); + } + + @Test + public void testAggregateOverStructuredPropertyV1() { + SearchConfiguration config = new SearchConfiguration(); + config.setMaxTermBucketSize(25); + + AggregationQueryBuilder builder = + new AggregationQueryBuilder( + config, ImmutableMap.of(mock(EntitySpec.class), ImmutableList.of())); + + List aggs = + builder.getAggregations( + TestOperationContexts.systemContextNoSearchAuthorization(aspectRetrieverV1), + List.of("structuredProperties.ab.fgh.ten")); + Assert.assertEquals(aggs.size(), 1); + AggregationBuilder aggBuilder = aggs.get(0); + Assert.assertTrue(aggBuilder instanceof TermsAggregationBuilder); + TermsAggregationBuilder agg = (TermsAggregationBuilder) aggBuilder; + // Check that field name is sanitized to correct field name + Assert.assertEquals( + agg.field(), + "structuredProperties._versioned.ab_fgh_ten.00000000000001.string.keyword", + "Terms aggregation must be on a keyword field or subfield."); + + // Two structured properties + aggs = + builder.getAggregations( + TestOperationContexts.systemContextNoSearchAuthorization(aspectRetrieverV1), + List.of( + "structuredProperties.ab.fgh.ten", + "structuredProperties._versioned.hello.00000000000001.string")); + Assert.assertEquals(aggs.size(), 2); + Assert.assertEquals( + aggs.stream() + .map(aggr -> ((TermsAggregationBuilder) aggr).field()) + .collect(Collectors.toSet()), + Set.of( + "structuredProperties._versioned.ab_fgh_ten.00000000000001.string.keyword", + "structuredProperties._versioned.hello.00000000000001.string.keyword")); } @Test @@ -240,7 +356,76 @@ public void testAggregateOverFieldsAndStructProp() { // Aggregate over fields and structured properties List aggs = builder.getAggregations( - TestOperationContexts.systemContextNoSearchAuthorization(), + TestOperationContexts.systemContextNoSearchAuthorization(aspectRetriever), + ImmutableList.of( + "test1", + "test2", + "hasTest1", + "structuredProperties.ab.fgh.ten", + "structuredProperties.hello")); + Assert.assertEquals(aggs.size(), 5); + Set facets = + aggs.stream() + .map(aggB -> ((TermsAggregationBuilder) aggB).field()) + .collect(Collectors.toSet()); + Assert.assertEquals( + facets, + ImmutableSet.of( + "test1.keyword", + "test2.keyword", + "hasTest1", + "structuredProperties.ab_fgh_ten.keyword", + "structuredProperties.hello.keyword")); + } + + @Test + public void testAggregateOverFieldsAndStructPropV1() { + SearchableAnnotation annotation1 = + new SearchableAnnotation( + "test1", + SearchableAnnotation.FieldType.KEYWORD, + true, + true, + false, + false, + Optional.empty(), + Optional.of("Has Test"), + 1.0, + Optional.of("hasTest1"), + Optional.empty(), + Collections.emptyMap(), + Collections.emptyList(), + false); + + SearchableAnnotation annotation2 = + new SearchableAnnotation( + "test2", + SearchableAnnotation.FieldType.KEYWORD, + true, + true, + false, + false, + Optional.of("Test Filter"), + Optional.empty(), + 1.0, + Optional.empty(), + Optional.empty(), + Collections.emptyMap(), + Collections.emptyList(), + false); + + SearchConfiguration config = new SearchConfiguration(); + config.setMaxTermBucketSize(25); + + AggregationQueryBuilder builder = + new AggregationQueryBuilder( + config, + ImmutableMap.of(mock(EntitySpec.class), ImmutableList.of(annotation1, annotation2))); + + // Aggregate over fields and structured properties + List aggs = + builder.getAggregations( + TestOperationContexts.systemContextNoSearchAuthorization(aspectRetrieverV1), ImmutableList.of( "test1", "test2", @@ -258,8 +443,8 @@ public void testAggregateOverFieldsAndStructProp() { "test1.keyword", "test2.keyword", "hasTest1", - "structuredProperties.ab_fgh_ten", - "structuredProperties.hello")); + "structuredProperties._versioned.ab_fgh_ten.00000000000001.string.keyword", + "structuredProperties._versioned.hello.00000000000001.string.keyword")); } @Test diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java index 2f68f17dae241..9376552f7abc5 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AutocompleteRequestHandlerTest.java @@ -5,7 +5,6 @@ import static org.testng.Assert.assertTrue; import com.linkedin.metadata.TestEntitySpecBuilder; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.config.search.custom.AutocompleteConfiguration; import com.linkedin.metadata.config.search.custom.BoolQueryConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; @@ -34,9 +33,7 @@ public class AutocompleteRequestHandlerTest { private AutocompleteRequestHandler handler = AutocompleteRequestHandler.getBuilder( - TestEntitySpecBuilder.getSpec(), - CustomSearchConfiguration.builder().build(), - TestOperationContexts.emptyAspectRetriever(null)); + TestEntitySpecBuilder.getSpec(), CustomSearchConfiguration.builder().build()); private OperationContext mockOpContext = TestOperationContexts.systemContextNoSearchAuthorization(mock(EntityRegistry.class)); @@ -173,8 +170,7 @@ public void testCustomConfigWithDefault() { .should(List.of(Map.of("match_all", Map.of()))) .build()) .build())) - .build(), - mock(AspectRetriever.class)); + .build()); SearchRequest autocompleteRequest = withoutDefaultQuery.getSearchRequest(mockOpContext, "input", null, null, 10); @@ -199,8 +195,7 @@ public void testCustomConfigWithDefault() { .should(List.of(Map.of("match_all", Map.of()))) .build()) .build())) - .build(), - mock(AspectRetriever.class)); + .build()); autocompleteRequest = withDefaultQuery.getSearchRequest(mockOpContext, "input", null, null, 10); sourceBuilder = autocompleteRequest.source(); @@ -242,8 +237,7 @@ public void testCustomConfigWithInheritedQueryFunctionScores() { .should(List.of(Map.of("match_all", Map.of()))) .build()) .build())) - .build(), - mock(AspectRetriever.class)); + .build()); SearchRequest autocompleteRequest = withInherit.getSearchRequest(mockOpContext, "input", null, null, 10); @@ -281,8 +275,7 @@ public void testCustomConfigWithInheritedQueryFunctionScores() { .should(List.of(Map.of("match_all", Map.of()))) .build()) .build())) - .build(), - mock(AspectRetriever.class)); + .build()); autocompleteRequest = noQueryCustomization.getSearchRequest(mockOpContext, "input", null, null, 10); @@ -344,8 +337,7 @@ public void testCustomConfigWithFunctionScores() { Map.of( "deprecated", Map.of("value", false))))))) .build())) - .build(), - mock(AspectRetriever.class)); + .build()); SearchRequest autocompleteRequest = explicitNoInherit.getSearchRequest(mockOpContext, "input", null, null, 10); @@ -397,8 +389,7 @@ public void testCustomConfigWithFunctionScores() { Map.of( "deprecated", Map.of("value", false))))))) .build())) - .build(), - mock(AspectRetriever.class)); + .build()); autocompleteRequest = explicit.getSearchRequest(mockOpContext, "input", null, null, 10); sourceBuilder = autocompleteRequest.source(); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java index 1b41ff44bc969..1cd9a274463d3 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java @@ -86,11 +86,7 @@ public class SearchRequestHandlerTest extends AbstractTestNGSpringContextTests { public void testDatasetFieldsAndHighlights() { EntitySpec entitySpec = operationContext.getEntityRegistry().getEntitySpec("dataset"); SearchRequestHandler datasetHandler = - SearchRequestHandler.getBuilder( - entitySpec, - testQueryConfig, - null, - operationContext.getRetrieverContext().get().getAspectRetriever()); + SearchRequestHandler.getBuilder(entitySpec, testQueryConfig, null); /* Ensure efficient query performance, we do not expect upstream/downstream/fineGrained lineage @@ -109,11 +105,7 @@ public void testDatasetFieldsAndHighlights() { @Test public void testSearchRequestHandlerHighlightingTurnedOff() { SearchRequestHandler requestHandler = - SearchRequestHandler.getBuilder( - TestEntitySpecBuilder.getSpec(), - testQueryConfig, - null, - operationContext.getRetrieverContext().get().getAspectRetriever()); + SearchRequestHandler.getBuilder(TestEntitySpecBuilder.getSpec(), testQueryConfig, null); SearchRequest searchRequest = requestHandler.getSearchRequest( operationContext.withSearchFlags( @@ -153,11 +145,7 @@ public void testSearchRequestHandlerHighlightingTurnedOff() { @Test public void testSearchRequestHandler() { SearchRequestHandler requestHandler = - SearchRequestHandler.getBuilder( - TestEntitySpecBuilder.getSpec(), - testQueryConfig, - null, - operationContext.getRetrieverContext().get().getAspectRetriever()); + SearchRequestHandler.getBuilder(TestEntitySpecBuilder.getSpec(), testQueryConfig, null); SearchRequest searchRequest = requestHandler.getSearchRequest( operationContext.withSearchFlags( @@ -220,11 +208,7 @@ public void testSearchRequestHandler() { @Test public void testAggregationsInSearch() { SearchRequestHandler requestHandler = - SearchRequestHandler.getBuilder( - TestEntitySpecBuilder.getSpec(), - testQueryConfig, - null, - operationContext.getRetrieverContext().get().getAspectRetriever()); + SearchRequestHandler.getBuilder(TestEntitySpecBuilder.getSpec(), testQueryConfig, null); final String nestedAggString = String.format("_entityType%stextFieldOverride", AGGREGATION_SEPARATOR_CHAR); SearchRequest searchRequest = @@ -292,11 +276,7 @@ public void testAggregationsInSearch() { public void testFilteredSearch() { final SearchRequestHandler requestHandler = - SearchRequestHandler.getBuilder( - TestEntitySpecBuilder.getSpec(), - testQueryConfig, - null, - operationContext.getRetrieverContext().get().getAspectRetriever()); + SearchRequestHandler.getBuilder(TestEntitySpecBuilder.getSpec(), testQueryConfig, null); final BoolQueryBuilder testQuery = constructFilterQuery(requestHandler, false); @@ -675,11 +655,7 @@ private BoolQueryBuilder getQuery(final Criterion filterCriterion) { .setAnd(new CriterionArray(ImmutableList.of(filterCriterion))))); final SearchRequestHandler requestHandler = - SearchRequestHandler.getBuilder( - TestEntitySpecBuilder.getSpec(), - testQueryConfig, - null, - operationContext.getRetrieverContext().get().getAspectRetriever()); + SearchRequestHandler.getBuilder(TestEntitySpecBuilder.getSpec(), testQueryConfig, null); return (BoolQueryBuilder) requestHandler diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java index 9953e08efb2d2..def14f9be7054 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java @@ -21,6 +21,7 @@ import com.linkedin.metadata.TestEntityUtil; import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.GraphRetriever; +import com.linkedin.metadata.entity.SearchRetriever; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.SearchableRefFieldSpec; import com.linkedin.metadata.models.registry.ConfigEntityRegistry; @@ -186,6 +187,7 @@ public void testSetSearchableRefValue() throws URISyntaxException, RemoteInvocat RetrieverContext.builder() .aspectRetriever(aspectRetriever) .graphRetriever(mock(GraphRetriever.class)) + .searchRetriever(mock(SearchRetriever.class)) .build()); searchDocumentTransformer.setSearchableRefValue( @@ -241,6 +243,7 @@ public void testSetSearchableRefValue_RuntimeException() RetrieverContext.builder() .aspectRetriever(aspectRetriever) .graphRetriever(mock(GraphRetriever.class)) + .searchRetriever(mock(SearchRetriever.class)) .build()); ObjectNode searchDocument = JsonNodeFactory.instance.objectNode(); @@ -277,6 +280,7 @@ public void testSetSearchableRefValue_RuntimeException_URNExist() RetrieverContext.builder() .aspectRetriever(aspectRetriever) .graphRetriever(mock(GraphRetriever.class)) + .searchRetriever(mock(SearchRetriever.class)) .build()); ObjectNode searchDocument = JsonNodeFactory.instance.objectNode(); @@ -309,6 +313,7 @@ void testSetSearchableRefValue_WithInvalidURN() RetrieverContext.builder() .aspectRetriever(aspectRetriever) .graphRetriever(mock(GraphRetriever.class)) + .searchRetriever(mock(SearchRetriever.class)) .build()); ObjectNode searchDocument = JsonNodeFactory.instance.objectNode(); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java index 838df98fdce9c..d56d9b0674884 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java @@ -1,18 +1,81 @@ package com.linkedin.metadata.search.utils; +import static com.linkedin.metadata.Constants.DATA_TYPE_URN_PREFIX; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.SetMode; import com.linkedin.data.template.StringArray; +import com.linkedin.entity.Aspect; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.structured.StructuredPropertyDefinition; +import io.datahubproject.test.metadata.context.TestOperationContexts; +import java.net.URISyntaxException; import java.util.HashMap; +import java.util.Map; +import java.util.Set; import org.opensearch.index.query.QueryBuilder; import org.testng.Assert; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; public class ESUtilsTest { private static final String FIELD_TO_EXPAND = "fieldTags"; + private static AspectRetriever aspectRetriever; + private static AspectRetriever aspectRetrieverV1; + + @BeforeClass + public static void setup() throws RemoteInvocationException, URISyntaxException { + Urn abFghTenUrn = Urn.createFromString("urn:li:structuredProperty:ab.fgh.ten"); + + // legacy + aspectRetriever = mock(AspectRetriever.class); + when(aspectRetriever.getEntityRegistry()) + .thenReturn(TestOperationContexts.defaultEntityRegistry()); + + StructuredPropertyDefinition structPropAbFghTenDefinition = new StructuredPropertyDefinition(); + structPropAbFghTenDefinition.setVersion(null, SetMode.REMOVE_IF_NULL); + structPropAbFghTenDefinition.setValueType( + Urn.createFromString(DATA_TYPE_URN_PREFIX + "string")); + structPropAbFghTenDefinition.setQualifiedName("ab.fgh.ten"); + when(aspectRetriever.getLatestAspectObjects(eq(Set.of(abFghTenUrn)), anySet())) + .thenReturn( + Map.of( + abFghTenUrn, + Map.of( + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, + new Aspect(structPropAbFghTenDefinition.data())))); + + // V1 + aspectRetrieverV1 = mock(AspectRetriever.class); + when(aspectRetrieverV1.getEntityRegistry()) + .thenReturn(TestOperationContexts.defaultEntityRegistry()); + + StructuredPropertyDefinition structPropAbFghTenDefinitionV1 = + new StructuredPropertyDefinition(); + structPropAbFghTenDefinitionV1.setVersion("00000000000001"); + structPropAbFghTenDefinitionV1.setValueType( + Urn.createFromString(DATA_TYPE_URN_PREFIX + "string")); + structPropAbFghTenDefinitionV1.setQualifiedName("ab.fgh.ten"); + when(aspectRetrieverV1.getLatestAspectObjects(eq(Set.of(abFghTenUrn)), anySet())) + .thenReturn( + Map.of( + abFghTenUrn, + Map.of( + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, + new Aspect(structPropAbFghTenDefinitionV1.data())))); + } + @Test public void testGetQueryBuilderFromCriterionEqualsValues() { @@ -23,7 +86,8 @@ public void testGetQueryBuilderFromCriterionEqualsValues() { .setValues(new StringArray(ImmutableList.of("value1"))); QueryBuilder result = - ESUtils.getQueryBuilderFromCriterion(singleValueCriterion, false, new HashMap<>()); + ESUtils.getQueryBuilderFromCriterion( + singleValueCriterion, false, new HashMap<>(), mock(AspectRetriever.class)); String expected = "{\n" + " \"terms\" : {\n" @@ -42,7 +106,9 @@ public void testGetQueryBuilderFromCriterionEqualsValues() { .setCondition(Condition.EQUAL) .setValues(new StringArray(ImmutableList.of("value1", "value2"))); - result = ESUtils.getQueryBuilderFromCriterion(multiValueCriterion, false, new HashMap<>()); + result = + ESUtils.getQueryBuilderFromCriterion( + multiValueCriterion, false, new HashMap<>(), mock(AspectRetriever.class)); expected = "{\n" + " \"terms\" : {\n" @@ -62,7 +128,9 @@ public void testGetQueryBuilderFromCriterionEqualsValues() { .setCondition(Condition.EQUAL) .setValues(new StringArray(ImmutableList.of("value1", "value2"))); - result = ESUtils.getQueryBuilderFromCriterion(timeseriesField, true, new HashMap<>()); + result = + ESUtils.getQueryBuilderFromCriterion( + timeseriesField, true, new HashMap<>(), mock(AspectRetriever.class)); expected = "{\n" + " \"terms\" : {\n" @@ -83,7 +151,8 @@ public void testGetQueryBuilderFromCriterionExists() { new Criterion().setField("myTestField").setCondition(Condition.EXISTS); QueryBuilder result = - ESUtils.getQueryBuilderFromCriterion(singleValueCriterion, false, new HashMap<>()); + ESUtils.getQueryBuilderFromCriterion( + singleValueCriterion, false, new HashMap<>(), mock(AspectRetriever.class)); String expected = "{\n" + " \"bool\" : {\n" @@ -106,7 +175,9 @@ public void testGetQueryBuilderFromCriterionExists() { final Criterion timeseriesField = new Criterion().setField("myTestField").setCondition(Condition.EXISTS); - result = ESUtils.getQueryBuilderFromCriterion(timeseriesField, true, new HashMap<>()); + result = + ESUtils.getQueryBuilderFromCriterion( + timeseriesField, true, new HashMap<>(), mock(AspectRetriever.class)); expected = "{\n" + " \"bool\" : {\n" @@ -132,7 +203,8 @@ public void testGetQueryBuilderFromCriterionIsNull() { new Criterion().setField("myTestField").setCondition(Condition.IS_NULL); QueryBuilder result = - ESUtils.getQueryBuilderFromCriterion(singleValueCriterion, false, new HashMap<>()); + ESUtils.getQueryBuilderFromCriterion( + singleValueCriterion, false, new HashMap<>(), mock(AspectRetriever.class)); String expected = "{\n" + " \"bool\" : {\n" @@ -155,7 +227,9 @@ public void testGetQueryBuilderFromCriterionIsNull() { final Criterion timeseriesField = new Criterion().setField("myTestField").setCondition(Condition.IS_NULL); - result = ESUtils.getQueryBuilderFromCriterion(timeseriesField, true, new HashMap<>()); + result = + ESUtils.getQueryBuilderFromCriterion( + timeseriesField, true, new HashMap<>(), mock(AspectRetriever.class)); expected = "{\n" + " \"bool\" : {\n" @@ -187,7 +261,8 @@ public void testGetQueryBuilderFromCriterionFieldToExpand() { // Ensure that the query is expanded! QueryBuilder result = - ESUtils.getQueryBuilderFromCriterion(singleValueCriterion, false, new HashMap<>()); + ESUtils.getQueryBuilderFromCriterion( + singleValueCriterion, false, new HashMap<>(), mock(AspectRetriever.class)); String expected = "{\n" + " \"bool\" : {\n" @@ -225,7 +300,9 @@ public void testGetQueryBuilderFromCriterionFieldToExpand() { .setValues(new StringArray(ImmutableList.of("value1", "value2"))); // Ensure that the query is expanded without keyword. - result = ESUtils.getQueryBuilderFromCriterion(timeseriesField, true, new HashMap<>()); + result = + ESUtils.getQueryBuilderFromCriterion( + timeseriesField, true, new HashMap<>(), mock(AspectRetriever.class)); expected = "{\n" + " \"bool\" : {\n" @@ -268,15 +345,41 @@ public void testGetQueryBuilderFromStructPropEqualsValue() { .setValues(new StringArray(ImmutableList.of("value1"))); QueryBuilder result = - ESUtils.getQueryBuilderFromCriterion(singleValueCriterion, false, new HashMap<>()); + ESUtils.getQueryBuilderFromCriterion( + singleValueCriterion, false, new HashMap<>(), aspectRetriever); + String expected = + "{\n" + + " \"terms\" : {\n" + + " \"structuredProperties.ab_fgh_ten.keyword\" : [\n" + + " \"value1\"\n" + + " ],\n" + + " \"boost\" : 1.0,\n" + + " \"_name\" : \"structuredProperties.ab.fgh.ten\"\n" + + " }\n" + + "}"; + Assert.assertEquals(result.toString(), expected); + } + + @Test + public void testGetQueryBuilderFromStructPropEqualsValueV1() { + + final Criterion singleValueCriterion = + new Criterion() + .setField("structuredProperties.ab.fgh.ten") + .setCondition(Condition.EQUAL) + .setValues(new StringArray(ImmutableList.of("value1"))); + + QueryBuilder result = + ESUtils.getQueryBuilderFromCriterion( + singleValueCriterion, false, new HashMap<>(), aspectRetrieverV1); String expected = "{\n" + " \"terms\" : {\n" - + " \"structuredProperties.ab_fgh_ten\" : [\n" + + " \"structuredProperties._versioned.ab_fgh_ten.00000000000001.string.keyword\" : [\n" + " \"value1\"\n" + " ],\n" + " \"boost\" : 1.0,\n" - + " \"_name\" : \"structuredProperties.ab_fgh_ten\"\n" + + " \"_name\" : \"structuredProperties.ab.fgh.ten\"\n" + " }\n" + "}"; Assert.assertEquals(result.toString(), expected); @@ -288,7 +391,8 @@ public void testGetQueryBuilderFromStructPropExists() { new Criterion().setField("structuredProperties.ab.fgh.ten").setCondition(Condition.EXISTS); QueryBuilder result = - ESUtils.getQueryBuilderFromCriterion(singleValueCriterion, false, new HashMap<>()); + ESUtils.getQueryBuilderFromCriterion( + singleValueCriterion, false, new HashMap<>(), aspectRetriever); String expected = "{\n" + " \"bool\" : {\n" @@ -302,7 +406,59 @@ public void testGetQueryBuilderFromStructPropExists() { + " ],\n" + " \"adjust_pure_negative\" : true,\n" + " \"boost\" : 1.0,\n" - + " \"_name\" : \"structuredProperties.ab_fgh_ten\"\n" + + " \"_name\" : \"structuredProperties.ab.fgh.ten\"\n" + + " }\n" + + "}"; + Assert.assertEquals(result.toString(), expected); + + // No diff in the timeseries field case for this condition. + final Criterion timeseriesField = + new Criterion().setField("myTestField").setCondition(Condition.EXISTS); + + result = + ESUtils.getQueryBuilderFromCriterion( + timeseriesField, true, new HashMap<>(), aspectRetriever); + expected = + "{\n" + + " \"bool\" : {\n" + + " \"must\" : [\n" + + " {\n" + + " \"exists\" : {\n" + + " \"field\" : \"myTestField\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + " ],\n" + + " \"adjust_pure_negative\" : true,\n" + + " \"boost\" : 1.0,\n" + + " \"_name\" : \"myTestField\"\n" + + " }\n" + + "}"; + Assert.assertEquals(result.toString(), expected); + } + + @Test + public void testGetQueryBuilderFromStructPropExistsV1() { + final Criterion singleValueCriterion = + new Criterion().setField("structuredProperties.ab.fgh.ten").setCondition(Condition.EXISTS); + + QueryBuilder result = + ESUtils.getQueryBuilderFromCriterion( + singleValueCriterion, false, new HashMap<>(), aspectRetrieverV1); + String expected = + "{\n" + + " \"bool\" : {\n" + + " \"must\" : [\n" + + " {\n" + + " \"exists\" : {\n" + + " \"field\" : \"structuredProperties._versioned.ab_fgh_ten.00000000000001.string\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + " ],\n" + + " \"adjust_pure_negative\" : true,\n" + + " \"boost\" : 1.0,\n" + + " \"_name\" : \"structuredProperties.ab.fgh.ten\"\n" + " }\n" + "}"; Assert.assertEquals(result.toString(), expected); @@ -311,7 +467,9 @@ public void testGetQueryBuilderFromStructPropExists() { final Criterion timeseriesField = new Criterion().setField("myTestField").setCondition(Condition.EXISTS); - result = ESUtils.getQueryBuilderFromCriterion(timeseriesField, true, new HashMap<>()); + result = + ESUtils.getQueryBuilderFromCriterion( + timeseriesField, true, new HashMap<>(), aspectRetrieverV1); expected = "{\n" + " \"bool\" : {\n" diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/AssertionServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/service/AssertionServiceTest.java new file mode 100644 index 0000000000000..0390e4d8e0a58 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/service/AssertionServiceTest.java @@ -0,0 +1,316 @@ +package com.linkedin.metadata.service; + +import static com.linkedin.metadata.Constants.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import com.google.common.collect.ImmutableList; +import com.linkedin.assertion.AssertionInfo; +import com.linkedin.assertion.AssertionResult; +import com.linkedin.assertion.AssertionResultError; +import com.linkedin.assertion.AssertionResultErrorType; +import com.linkedin.assertion.AssertionResultType; +import com.linkedin.assertion.AssertionRunEvent; +import com.linkedin.assertion.AssertionRunStatus; +import com.linkedin.assertion.AssertionSourceType; +import com.linkedin.assertion.AssertionType; +import com.linkedin.assertion.CustomAssertionInfo; +import com.linkedin.common.DataPlatformInstance; +import com.linkedin.common.EntityRelationship; +import com.linkedin.common.EntityRelationshipArray; +import com.linkedin.common.EntityRelationships; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringMap; +import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.metadata.graph.GraphClient; +import com.linkedin.metadata.query.filter.RelationshipDirection; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; +import io.datahubproject.metadata.context.OperationContext; +import io.datahubproject.test.metadata.context.TestOperationContexts; +import java.util.List; +import java.util.Map; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +public class AssertionServiceTest { + private static final Urn TEST_ACTOR_URN = UrnUtils.getUrn("urn:li:corpuser:test"); + private static final Urn TEST_ASSERTION_URN = UrnUtils.getUrn("urn:li:assertion:test"); + private static final Urn TEST_DATASET_URN = + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,name,PROD)"); + private static final Urn TEST_FIELD_URN = + UrnUtils.getUrn( + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,name,PROD),field1)"); + private static final Urn TEST_PLATFORM_URN = UrnUtils.getUrn("urn:li:dataPlatform:hive"); + + private static final Urn TEST_PLATFORM_INSTANCE_URN = + UrnUtils.getUrn("urn:li:dataPlatformInstance:(urn:li:dataPlatform:custom,instance1)"); + + private OperationContext opContext; + + @BeforeTest + public void setup() { + opContext = TestOperationContexts.userContextNoSearchAuthorization(TEST_ACTOR_URN); + } + + @Test + public void testUpsertCustomAssertionRequiredFields() + throws Exception, RemoteInvocationException { + SystemEntityClient mockedEntityClient = mock(SystemEntityClient.class); + AssertionService assertionService = + new AssertionService(mockedEntityClient, mock(GraphClient.class)); + + String descriptionOfCustomAssertion = "Description of custom assertion"; + Mockito.doAnswer( + invocation -> { + List aspects = invocation.getArgument(1); + Assert.assertEquals(aspects.size(), 2); + + MetadataChangeProposal proposal1 = aspects.get(0); + Assert.assertEquals(proposal1.getEntityUrn(), TEST_ASSERTION_URN); + Assert.assertEquals(proposal1.getAspectName(), ASSERTION_INFO_ASPECT_NAME); + AssertionInfo info = + GenericRecordUtils.deserializeAspect( + proposal1.getAspect().getValue(), + proposal1.getAspect().getContentType(), + AssertionInfo.class); + + Assert.assertEquals(info.getType(), AssertionType.CUSTOM); + Assert.assertEquals(info.getDescription(), descriptionOfCustomAssertion); + Assert.assertEquals(info.getSource().getType(), AssertionSourceType.EXTERNAL); + CustomAssertionInfo customAssertionInfo = info.getCustomAssertion(); + Assert.assertEquals(customAssertionInfo.getEntity(), TEST_DATASET_URN); + + MetadataChangeProposal proposal2 = aspects.get(1); + Assert.assertEquals(proposal2.getEntityUrn(), TEST_ASSERTION_URN); + Assert.assertEquals(proposal2.getAspectName(), DATA_PLATFORM_INSTANCE_ASPECT_NAME); + DataPlatformInstance dataPlatformInstance = + GenericRecordUtils.deserializeAspect( + proposal2.getAspect().getValue(), + proposal2.getAspect().getContentType(), + DataPlatformInstance.class); + Assert.assertEquals(dataPlatformInstance.getPlatform(), TEST_PLATFORM_URN); + return null; + }) + .when(mockedEntityClient) + .batchIngestProposals(any(OperationContext.class), Mockito.anyList(), Mockito.eq(false)); + + Urn urn = + assertionService.upsertCustomAssertion( + opContext, + TEST_ASSERTION_URN, + TEST_DATASET_URN, + descriptionOfCustomAssertion, + null, + new DataPlatformInstance().setPlatform(TEST_PLATFORM_URN), + new CustomAssertionInfo().setEntity(TEST_DATASET_URN)); + Assert.assertEquals(urn.getEntityType(), "assertion"); + + Mockito.verify(mockedEntityClient, Mockito.times(1)) + .batchIngestProposals(any(OperationContext.class), Mockito.anyList(), Mockito.eq(false)); + } + + @Test + public void testUpsertCustomAssertionAllFields() throws Exception, RemoteInvocationException { + SystemEntityClient mockedEntityClient = mock(SystemEntityClient.class); + AssertionService assertionService = + new AssertionService(mockedEntityClient, mock(GraphClient.class)); + String descriptionOfCustomAssertion = "Description of custom assertion"; + String externalUrlOfCustomAssertion = "https://xyz.com/abc"; + String customCategory = "Custom category"; + String customLogic = "select percentile(field1, 0.66) from table"; + Mockito.doAnswer( + invocation -> { + List aspects = invocation.getArgument(1); + Assert.assertEquals(aspects.size(), 2); + + MetadataChangeProposal proposal1 = aspects.get(0); + Assert.assertEquals(proposal1.getEntityUrn(), TEST_ASSERTION_URN); + Assert.assertEquals(proposal1.getAspectName(), ASSERTION_INFO_ASPECT_NAME); + AssertionInfo info = + GenericRecordUtils.deserializeAspect( + proposal1.getAspect().getValue(), + proposal1.getAspect().getContentType(), + AssertionInfo.class); + + Assert.assertEquals(info.getType(), AssertionType.CUSTOM); + Assert.assertEquals(info.getDescription(), descriptionOfCustomAssertion); + Assert.assertEquals(info.getExternalUrl().toString(), externalUrlOfCustomAssertion); + Assert.assertEquals(info.getSource().getType(), AssertionSourceType.EXTERNAL); + CustomAssertionInfo customAssertionInfo = info.getCustomAssertion(); + Assert.assertEquals(customAssertionInfo.getEntity(), TEST_DATASET_URN); + Assert.assertEquals(customAssertionInfo.getField(), TEST_FIELD_URN); + Assert.assertEquals(customAssertionInfo.getType(), customCategory); + Assert.assertEquals(customAssertionInfo.getLogic(), customLogic); + + MetadataChangeProposal proposal2 = aspects.get(1); + Assert.assertEquals(proposal2.getEntityUrn(), TEST_ASSERTION_URN); + Assert.assertEquals(proposal2.getAspectName(), DATA_PLATFORM_INSTANCE_ASPECT_NAME); + DataPlatformInstance dataPlatformInstance = + GenericRecordUtils.deserializeAspect( + proposal2.getAspect().getValue(), + proposal2.getAspect().getContentType(), + DataPlatformInstance.class); + Assert.assertEquals(dataPlatformInstance.getPlatform(), TEST_PLATFORM_URN); + Assert.assertEquals(dataPlatformInstance.getInstance(), TEST_PLATFORM_INSTANCE_URN); + return null; + }) + .when(mockedEntityClient) + .batchIngestProposals(any(OperationContext.class), Mockito.anyList(), Mockito.eq(false)); + + Urn urn = + assertionService.upsertCustomAssertion( + opContext, + TEST_ASSERTION_URN, + TEST_DATASET_URN, + descriptionOfCustomAssertion, + externalUrlOfCustomAssertion, + new DataPlatformInstance() + .setPlatform(TEST_PLATFORM_URN) + .setInstance(TEST_PLATFORM_INSTANCE_URN), + new CustomAssertionInfo() + .setEntity(TEST_DATASET_URN) + .setField(TEST_FIELD_URN) + .setType(customCategory) + .setLogic(customLogic)); + Assert.assertEquals(urn.getEntityType(), "assertion"); + + Mockito.verify(mockedEntityClient, Mockito.times(1)) + .batchIngestProposals(any(OperationContext.class), Mockito.anyList(), Mockito.eq(false)); + } + + @Test + public void testAddAssertionRunEventRequiredFields() throws Exception, RemoteInvocationException { + SystemEntityClient mockedEntityClient = mock(SystemEntityClient.class); + AssertionService assertionService = + new AssertionService(mockedEntityClient, mock(GraphClient.class)); + Long eventtime = 1718619000000L; + + Mockito.doAnswer( + invocation -> { + MetadataChangeProposal proposal = invocation.getArgument(1); + + Assert.assertEquals(proposal.getEntityUrn(), TEST_ASSERTION_URN); + Assert.assertEquals(proposal.getAspectName(), ASSERTION_RUN_EVENT_ASPECT_NAME); + AssertionRunEvent runEvent = + GenericRecordUtils.deserializeAspect( + proposal.getAspect().getValue(), + proposal.getAspect().getContentType(), + AssertionRunEvent.class); + + Assert.assertEquals(runEvent.getAssertionUrn(), TEST_ASSERTION_URN); + Assert.assertEquals(runEvent.getAsserteeUrn(), TEST_DATASET_URN); + Assert.assertEquals(runEvent.getTimestampMillis(), eventtime); + Assert.assertEquals(runEvent.getStatus(), AssertionRunStatus.COMPLETE); + + AssertionResult result = runEvent.getResult(); + Assert.assertEquals(result.getType(), AssertionResultType.SUCCESS); + + return null; + }) + .when(mockedEntityClient) + .ingestProposal(any(OperationContext.class), Mockito.any(), Mockito.eq(false)); + + assertionService.addAssertionRunEvent( + opContext, + TEST_ASSERTION_URN, + TEST_DATASET_URN, + eventtime, + new AssertionResult().setType(AssertionResultType.SUCCESS)); + + Mockito.verify(mockedEntityClient, Mockito.times(1)) + .ingestProposal(any(OperationContext.class), Mockito.any(), Mockito.eq(false)); + } + + @Test + public void testAddAssertionRunEventAllFields() throws Exception, RemoteInvocationException { + SystemEntityClient mockedEntityClient = mock(SystemEntityClient.class); + AssertionService assertionService = + new AssertionService(mockedEntityClient, mock(GraphClient.class)); + Long eventtime = 1718619000000L; + StringMap nativeResults = new StringMap(Map.of("prop-1", "value-1")); + StringMap errorProps = new StringMap(Map.of("message", "errorMessage")); + String externalUrlOfAssertion = "https://abc/xyz"; + + Mockito.doAnswer( + invocation -> { + MetadataChangeProposal proposal = invocation.getArgument(1); + + Assert.assertEquals(proposal.getEntityUrn(), TEST_ASSERTION_URN); + Assert.assertEquals(proposal.getAspectName(), ASSERTION_RUN_EVENT_ASPECT_NAME); + AssertionRunEvent runEvent = + GenericRecordUtils.deserializeAspect( + proposal.getAspect().getValue(), + proposal.getAspect().getContentType(), + AssertionRunEvent.class); + + Assert.assertEquals(runEvent.getAssertionUrn(), TEST_ASSERTION_URN); + Assert.assertEquals(runEvent.getAsserteeUrn(), TEST_DATASET_URN); + Assert.assertEquals(runEvent.getTimestampMillis(), eventtime); + Assert.assertEquals(runEvent.getStatus(), AssertionRunStatus.COMPLETE); + Assert.assertEquals(runEvent.getResult().getNativeResults(), nativeResults); + + AssertionResult result = runEvent.getResult(); + Assert.assertEquals(result.getType(), AssertionResultType.ERROR); + Assert.assertEquals(result.getExternalUrl(), externalUrlOfAssertion); + Assert.assertEquals( + result.getError().getType(), AssertionResultErrorType.UNKNOWN_ERROR); + Assert.assertEquals(result.getError().getProperties(), errorProps); + + return null; + }) + .when(mockedEntityClient) + .ingestProposal(any(OperationContext.class), Mockito.any(), Mockito.eq(false)); + + assertionService.addAssertionRunEvent( + opContext, + TEST_ASSERTION_URN, + TEST_DATASET_URN, + eventtime, + new AssertionResult() + .setType(AssertionResultType.ERROR) + .setExternalUrl(externalUrlOfAssertion) + .setNativeResults(nativeResults) + .setError( + new AssertionResultError() + .setType(AssertionResultErrorType.UNKNOWN_ERROR) + .setProperties(errorProps))); + + Mockito.verify(mockedEntityClient, Mockito.times(1)) + .ingestProposal(any(OperationContext.class), Mockito.any(), Mockito.eq(false)); + } + + @Test + public void testGetEntityUrnForAssertion() throws Exception { + // Test data and mocks + SystemEntityClient mockClient = mock(SystemEntityClient.class); + GraphClient mockGraphClient = mock(GraphClient.class); + + Mockito.when( + mockGraphClient.getRelatedEntities( + Mockito.eq(TEST_ASSERTION_URN.toString()), + Mockito.eq(ImmutableList.of("Asserts")), + Mockito.eq(RelationshipDirection.OUTGOING), + Mockito.eq(0), + Mockito.eq(1), + Mockito.anyString())) + .thenReturn( + new EntityRelationships() + .setTotal(1) + .setRelationships( + new EntityRelationshipArray( + ImmutableList.of(new EntityRelationship().setEntity(TEST_DATASET_URN))))); + + final AssertionService service = new AssertionService(mockClient, mockGraphClient); + + // Test method + final Urn entityUrn = service.getEntityUrnForAssertion(opContext, TEST_ASSERTION_URN); + + // Assert result + Assert.assertEquals(entityUrn, TEST_DATASET_URN); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/hooks/PropertyDefinitionDeleteSideEffectTest.java b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/hooks/PropertyDefinitionDeleteSideEffectTest.java new file mode 100644 index 0000000000000..ab205d0463c4c --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/hooks/PropertyDefinitionDeleteSideEffectTest.java @@ -0,0 +1,193 @@ +package com.linkedin.metadata.structuredproperties.hooks; + +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_KEY_ASPECT_NAME; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; + +import com.linkedin.common.UrnArray; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.entity.Aspect; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.AspectRetriever; +import com.linkedin.metadata.aspect.batch.MCPItem; +import com.linkedin.metadata.aspect.batch.PatchMCP; +import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; +import com.linkedin.metadata.entity.SearchRetriever; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.ScrollResult; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.test.metadata.aspect.TestEntityRegistry; +import com.linkedin.test.metadata.aspect.batch.TestMCL; +import io.datahubproject.metadata.context.RetrieverContext; +import io.datahubproject.test.metadata.context.TestOperationContexts; +import jakarta.json.Json; +import jakarta.json.JsonPatch; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class PropertyDefinitionDeleteSideEffectTest { + private static final EntityRegistry TEST_REGISTRY = new TestEntityRegistry(); + private static final AspectPluginConfig TEST_PLUGIN_CONFIG = + AspectPluginConfig.builder() + .className(PropertyDefinitionDeleteSideEffect.class.getName()) + .enabled(true) + .supportedOperations(List.of("DELETE")) + .supportedEntityAspectNames( + List.of( + AspectPluginConfig.EntityAspectName.builder() + .entityName("structuredProperty") + .aspectName(STRUCTURED_PROPERTY_KEY_ASPECT_NAME) + .build(), + AspectPluginConfig.EntityAspectName.builder() + .entityName("structuredProperty") + .aspectName(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME) + .build())) + .build(); + + private static final Urn TEST_PROPERTY_URN = + UrnUtils.getUrn("urn:li:structuredProperty:io.acryl.privacy.retentionTime"); + private static final StructuredPropertyDefinition TEST_PROPERTY_DEFINITION = + new StructuredPropertyDefinition() + .setValueType(UrnUtils.getUrn("urn:li:type:datahub.string")) + .setVersion("00000000000001") + .setEntityTypes( + new UrnArray(List.of(UrnUtils.getUrn("urn:li:entityType:datahub.dataset")))) + .setQualifiedName("io.acryl.privacy.retentionTime"); + private static final Urn TEST_DATASET_URN = + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)"); + private AspectRetriever mockAspectRetriever; + private SearchRetriever mockSearchRetriever; + private RetrieverContext retrieverContext; + + @BeforeMethod + public void setup() { + mockAspectRetriever = mock(AspectRetriever.class); + when(mockAspectRetriever.getEntityRegistry()).thenReturn(TEST_REGISTRY); + when(mockAspectRetriever.getLatestAspectObject( + eq(TEST_PROPERTY_URN), eq(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME))) + .thenReturn(new Aspect(TEST_PROPERTY_DEFINITION.data())); + + mockSearchRetriever = mock(SearchRetriever.class); + ScrollResult scrollResult = new ScrollResult(); + scrollResult.setPageSize(1); + scrollResult.setNumEntities(1); + scrollResult.setEntities( + new SearchEntityArray(List.of(new SearchEntity().setEntity(TEST_DATASET_URN)))); + when(mockSearchRetriever.scroll( + eq(List.of("dataset")), eq(expectedFilter()), nullable(String.class), anyInt())) + .thenReturn(scrollResult); + + retrieverContext = + RetrieverContext.builder() + .searchRetriever(mockSearchRetriever) + .aspectRetriever(mockAspectRetriever) + .graphRetriever(TestOperationContexts.emptyGraphRetriever) + .build(); + } + + @Test + public void testDeletePropertyKey() { + PropertyDefinitionDeleteSideEffect test = new PropertyDefinitionDeleteSideEffect(); + test.setConfig(TEST_PLUGIN_CONFIG); + + List result = + test.postMCPSideEffect( + Set.of( + TestMCL.builder() + .changeType(ChangeType.DELETE) + .urn(TEST_PROPERTY_URN) + .entitySpec(TEST_REGISTRY.getEntitySpec("structuredProperty")) + .aspectSpec( + TEST_REGISTRY + .getEntitySpec("structuredProperty") + .getAspectSpec(STRUCTURED_PROPERTY_KEY_ASPECT_NAME)) + .build()), + retrieverContext) + .collect(Collectors.toList()); + + assertEquals(1, result.size()); + + verify(mockAspectRetriever, times(1)) + .getLatestAspectObject( + eq(TEST_PROPERTY_URN), eq(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)); + verify(mockSearchRetriever, times(1)) + .scroll(eq(List.of("dataset")), eq(expectedFilter()), nullable(String.class), anyInt()); + + JsonPatch expectedPatch = + Json.createPatchBuilder().remove("/properties/" + TEST_PROPERTY_URN).build(); + assertEquals(((PatchMCP) result.get(0)).getPatch(), expectedPatch); + } + + @Test + public void testDeletePropertyDefinition() { + PropertyDefinitionDeleteSideEffect test = new PropertyDefinitionDeleteSideEffect(); + test.setConfig(TEST_PLUGIN_CONFIG); + + List result = + test.postMCPSideEffect( + Set.of( + TestMCL.builder() + .changeType(ChangeType.DELETE) + .urn(TEST_PROPERTY_URN) + .entitySpec(TEST_REGISTRY.getEntitySpec("structuredProperty")) + .aspectSpec( + TEST_REGISTRY + .getEntitySpec("structuredProperty") + .getAspectSpec(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)) + .previousRecordTemplate(TEST_PROPERTY_DEFINITION) + .build()), + retrieverContext) + .collect(Collectors.toList()); + + assertEquals(1, result.size()); + + verify(mockAspectRetriever, times(0)).getLatestAspectObject(any(), any()); + verify(mockAspectRetriever, times(0)).getLatestAspectObjects(any(), any()); + verify(mockSearchRetriever, times(1)) + .scroll(eq(List.of("dataset")), eq(expectedFilter()), nullable(String.class), anyInt()); + + JsonPatch expectedPatch = + Json.createPatchBuilder().remove("/properties/" + TEST_PROPERTY_URN).build(); + assertEquals(((PatchMCP) result.get(0)).getPatch(), expectedPatch); + } + + private static Filter expectedFilter() { + Filter propertyFilter = new Filter(); + final ConjunctiveCriterionArray disjunction = new ConjunctiveCriterionArray(); + final ConjunctiveCriterion conjunction = new ConjunctiveCriterion(); + final CriterionArray andCriterion = new CriterionArray(); + + final Criterion propertyExistsCriterion = new Criterion(); + propertyExistsCriterion.setField( + "structuredProperties._versioned.io_acryl_privacy_retentionTime.00000000000001.string"); + propertyExistsCriterion.setCondition(Condition.EXISTS); + + andCriterion.add(propertyExistsCriterion); + conjunction.setAnd(andCriterion); + disjunction.add(conjunction); + propertyFilter.setOr(disjunction); + + return propertyFilter; + } +} diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/hooks/StructuredPropertiesSoftDeleteTest.java b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/hooks/StructuredPropertiesSoftDeleteTest.java similarity index 98% rename from entity-registry/src/test/java/com/linkedin/metadata/aspect/hooks/StructuredPropertiesSoftDeleteTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/hooks/StructuredPropertiesSoftDeleteTest.java index e1e84f5728540..9109eeb7f96a5 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/aspect/hooks/StructuredPropertiesSoftDeleteTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/hooks/StructuredPropertiesSoftDeleteTest.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata.aspect.hooks; +package com.linkedin.metadata.structuredproperties.hooks; import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; import static org.mockito.Mockito.mock; diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/PropertyDefinitionValidatorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java similarity index 70% rename from entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/PropertyDefinitionValidatorTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java index 841cbf5a77bec..22224f16f2210 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/PropertyDefinitionValidatorTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java @@ -1,34 +1,27 @@ -package com.linkedin.metadata.aspect.validators; +package com.linkedin.metadata.structuredproperties.validators; -import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; -import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.testng.Assert.*; +import static org.testng.AssertJUnit.assertEquals; import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; +import com.linkedin.data.template.SetMode; import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.GraphRetriever; import com.linkedin.metadata.aspect.RetrieverContext; -import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; -import com.linkedin.metadata.aspect.validation.PropertyDefinitionValidator; import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.structuredproperties.validation.PropertyDefinitionValidator; import com.linkedin.structured.PrimitivePropertyValue; import com.linkedin.structured.PropertyCardinality; import com.linkedin.structured.PropertyValue; import com.linkedin.structured.PropertyValueArray; import com.linkedin.structured.StructuredPropertyDefinition; -import com.linkedin.structured.StructuredPropertyKey; import com.linkedin.test.metadata.aspect.TestEntityRegistry; import com.linkedin.test.metadata.aspect.batch.TestMCP; import java.net.URISyntaxException; -import java.util.List; -import java.util.Set; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; @@ -109,6 +102,7 @@ public void testCannotChangeMultipleToSingle() oldProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); StructuredPropertyDefinition newProperty = oldProperty.copy(); newProperty.setCardinality(PropertyCardinality.SINGLE); + newProperty.setVersion(null, SetMode.REMOVE_IF_NULL); assertEquals( PropertyDefinitionValidator.validateDefinitionUpserts( TestMCP.ofOneMCP(testPropertyUrn, oldProperty, newProperty, entityRegistry), @@ -117,6 +111,30 @@ public void testCannotChangeMultipleToSingle() 1); } + @Test + public void testCanChangeMultipleToSingleWithNewVersion() + throws URISyntaxException, CloneNotSupportedException { + StructuredPropertyDefinition oldProperty = new StructuredPropertyDefinition(); + oldProperty.setEntityTypes( + new UrnArray( + Urn.createFromString("urn:li:logicalEntity:dataset"), + Urn.createFromString("urn:li:logicalEntity:chart"), + Urn.createFromString("urn:li:logicalEntity:glossaryTerm"))); + oldProperty.setDisplayName("oldProp"); + oldProperty.setQualifiedName("prop3"); + oldProperty.setCardinality(PropertyCardinality.MULTIPLE); + oldProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + StructuredPropertyDefinition newProperty = oldProperty.copy(); + newProperty.setCardinality(PropertyCardinality.SINGLE); + newProperty.setVersion("00000000000001"); + assertEquals( + PropertyDefinitionValidator.validateDefinitionUpserts( + TestMCP.ofOneMCP(testPropertyUrn, oldProperty, newProperty, entityRegistry), + mockRetrieverContext) + .count(), + 0); + } + @Test public void testCannotChangeValueType() throws URISyntaxException, CloneNotSupportedException { StructuredPropertyDefinition oldProperty = new StructuredPropertyDefinition(); @@ -131,6 +149,7 @@ public void testCannotChangeValueType() throws URISyntaxException, CloneNotSuppo oldProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); StructuredPropertyDefinition newProperty = oldProperty.copy(); newProperty.setValueType(Urn.createFromString("urn:li:logicalType:NUMBER")); + newProperty.setVersion(null, SetMode.REMOVE_IF_NULL); assertEquals( PropertyDefinitionValidator.validateDefinitionUpserts( TestMCP.ofOneMCP(testPropertyUrn, oldProperty, newProperty, entityRegistry), @@ -139,6 +158,30 @@ public void testCannotChangeValueType() throws URISyntaxException, CloneNotSuppo 1); } + @Test + public void testCanChangeValueTypeWithNewVersion() + throws URISyntaxException, CloneNotSupportedException { + StructuredPropertyDefinition oldProperty = new StructuredPropertyDefinition(); + oldProperty.setEntityTypes( + new UrnArray( + Urn.createFromString("urn:li:logicalEntity:dataset"), + Urn.createFromString("urn:li:logicalEntity:chart"), + Urn.createFromString("urn:li:logicalEntity:glossaryTerm"))); + oldProperty.setDisplayName("oldProp"); + oldProperty.setQualifiedName("prop3"); + oldProperty.setCardinality(PropertyCardinality.MULTIPLE); + oldProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + StructuredPropertyDefinition newProperty = oldProperty.copy(); + newProperty.setValueType(Urn.createFromString("urn:li:logicalType:NUMBER")); + newProperty.setVersion("00000000000001"); + assertEquals( + PropertyDefinitionValidator.validateDefinitionUpserts( + TestMCP.ofOneMCP(testPropertyUrn, oldProperty, newProperty, entityRegistry), + mockRetrieverContext) + .count(), + 0); + } + @Test public void testCanChangeDisplayName() throws URISyntaxException, CloneNotSupportedException, AspectValidationException { @@ -185,6 +228,30 @@ public void testCannotChangeFullyQualifiedName() 1); } + @Test + public void testCannotChangeFullyQualifiedNameWithVersionChange() + throws URISyntaxException, CloneNotSupportedException { + StructuredPropertyDefinition oldProperty = new StructuredPropertyDefinition(); + oldProperty.setEntityTypes( + new UrnArray( + Urn.createFromString("urn:li:logicalEntity:dataset"), + Urn.createFromString("urn:li:logicalEntity:chart"), + Urn.createFromString("urn:li:logicalEntity:glossaryTerm"))); + oldProperty.setDisplayName("oldProp"); + oldProperty.setQualifiedName("prop3"); + oldProperty.setCardinality(PropertyCardinality.MULTIPLE); + oldProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + StructuredPropertyDefinition newProperty = oldProperty.copy(); + newProperty.setQualifiedName("newProp"); + newProperty.setVersion("00000000000001"); + assertEquals( + PropertyDefinitionValidator.validateDefinitionUpserts( + TestMCP.ofOneMCP(testPropertyUrn, oldProperty, newProperty, entityRegistry), + mockRetrieverContext) + .count(), + 1); + } + @Test public void testCannotChangeRestrictAllowedValues() throws URISyntaxException, CloneNotSupportedException { @@ -203,6 +270,7 @@ public void testCannotChangeRestrictAllowedValues() PropertyValue allowedValue = new PropertyValue().setValue(PrimitivePropertyValue.create(1.0)).setDescription("hello"); newProperty.setAllowedValues(new PropertyValueArray(allowedValue)); + newProperty.setVersion(null, SetMode.REMOVE_IF_NULL); assertEquals( PropertyDefinitionValidator.validateDefinitionUpserts( TestMCP.ofOneMCP(testPropertyUrn, oldProperty, newProperty, entityRegistry), @@ -222,6 +290,46 @@ public void testCannotChangeRestrictAllowedValues() 1); } + @Test + public void testCanChangeRestrictAllowedValuesWithVersionChange() + throws URISyntaxException, CloneNotSupportedException { + // No constraint -> constraint case + StructuredPropertyDefinition oldProperty = new StructuredPropertyDefinition(); + oldProperty.setEntityTypes( + new UrnArray( + Urn.createFromString("urn:li:logicalEntity:dataset"), + Urn.createFromString("urn:li:logicalEntity:chart"), + Urn.createFromString("urn:li:logicalEntity:glossaryTerm"))); + oldProperty.setDisplayName("oldProp"); + oldProperty.setQualifiedName("prop3"); + oldProperty.setCardinality(PropertyCardinality.MULTIPLE); + oldProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + + StructuredPropertyDefinition newProperty = oldProperty.copy(); + newProperty.setVersion("00000000000001"); + PropertyValue allowedValue = + new PropertyValue().setValue(PrimitivePropertyValue.create(1.0)).setDescription("hello"); + newProperty.setAllowedValues(new PropertyValueArray(allowedValue)); + + assertEquals( + PropertyDefinitionValidator.validateDefinitionUpserts( + TestMCP.ofOneMCP(testPropertyUrn, oldProperty, newProperty, entityRegistry), + mockRetrieverContext) + .count(), + 0); + + // Remove allowed values from constraint case + PropertyValue oldAllowedValue = + new PropertyValue().setValue(PrimitivePropertyValue.create(3.0)).setDescription("hello"); + oldProperty.setAllowedValues((new PropertyValueArray(allowedValue, oldAllowedValue))); + assertEquals( + PropertyDefinitionValidator.validateDefinitionUpserts( + TestMCP.ofOneMCP(testPropertyUrn, oldProperty, newProperty, entityRegistry), + mockRetrieverContext) + .count(), + 0); + } + @Test public void testCanExpandAllowedValues() throws URISyntaxException, CloneNotSupportedException, AspectValidationException { @@ -289,61 +397,4 @@ public void testCanChangeAllowedValueDescriptions() .count(), 0); } - - @Test - public void testHardDeleteBlock() { - PropertyDefinitionValidator test = - new PropertyDefinitionValidator() - .setConfig( - AspectPluginConfig.builder() - .enabled(true) - .className(PropertyDefinitionValidator.class.getName()) - .supportedOperations(List.of("DELETE")) - .supportedEntityAspectNames( - List.of( - AspectPluginConfig.EntityAspectName.builder() - .entityName(STRUCTURED_PROPERTY_ENTITY_NAME) - .aspectName(Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME) - .build(), - AspectPluginConfig.EntityAspectName.builder() - .entityName(STRUCTURED_PROPERTY_ENTITY_NAME) - .aspectName("structuredPropertyKey") - .build())) - .build()); - - assertEquals( - test.validateProposed( - Set.of( - TestMCP.builder() - .changeType(ChangeType.DELETE) - .urn(UrnUtils.getUrn("urn:li:structuredProperty:foo.bar")) - .entitySpec(entityRegistry.getEntitySpec("structuredProperty")) - .aspectSpec( - entityRegistry - .getEntitySpec(STRUCTURED_PROPERTY_ENTITY_NAME) - .getKeyAspectSpec()) - .recordTemplate(new StructuredPropertyKey()) - .build()), - mockRetrieverContext) - .count(), - 1); - - assertEquals( - test.validateProposed( - Set.of( - TestMCP.builder() - .changeType(ChangeType.DELETE) - .urn(UrnUtils.getUrn("urn:li:structuredProperty:foo.bar")) - .entitySpec(entityRegistry.getEntitySpec("structuredProperty")) - .aspectSpec( - entityRegistry - .getEntitySpec(STRUCTURED_PROPERTY_ENTITY_NAME) - .getAspectSpecMap() - .get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)) - .recordTemplate(new StructuredPropertyDefinition()) - .build()), - mockRetrieverContext) - .count(), - 1); - } } diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/StructuredPropertiesValidatorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/StructuredPropertiesValidatorTest.java similarity index 98% rename from entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/StructuredPropertiesValidatorTest.java rename to metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/StructuredPropertiesValidatorTest.java index 77cf453f517be..0ab5b68327f2f 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/StructuredPropertiesValidatorTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/StructuredPropertiesValidatorTest.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata.aspect.validators; +package com.linkedin.metadata.structuredproperties.validators; import static org.testng.Assert.assertEquals; @@ -7,8 +7,8 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; -import com.linkedin.metadata.aspect.validation.StructuredPropertiesValidator; import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.structuredproperties.validation.StructuredPropertiesValidator; import com.linkedin.structured.PrimitivePropertyValue; import com.linkedin.structured.PrimitivePropertyValueArray; import com.linkedin.structured.PropertyValue; @@ -389,7 +389,7 @@ public void testValidateImmutableMutation() throws URISyntaxException { .collect(Collectors.toList()); Assert.assertEquals(exceptions.size(), 1, "Expected rejected mutation of immutable property."); - Assert.assertEquals(exceptions.get(0).getExceptionKey().getKey(), TEST_DATASET_URN); + Assert.assertEquals(exceptions.get(0).getAspectGroup().getKey(), TEST_DATASET_URN); Assert.assertTrue( exceptions.get(0).getMessage().contains("Cannot mutate an immutable property")); } @@ -484,7 +484,7 @@ public void testValidateImmutableDelete() throws URISyntaxException { .collect(Collectors.toList()); Assert.assertEquals(exceptions.size(), 1, "Expected rejected delete of immutable property."); - Assert.assertEquals(exceptions.get(0).getExceptionKey().getKey(), TEST_DATASET_URN); + Assert.assertEquals(exceptions.get(0).getAspectGroup().getKey(), TEST_DATASET_URN); Assert.assertTrue( exceptions.get(0).getMessage().contains("Cannot delete an immutable property")); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/systemmetadata/SystemMetadataServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/systemmetadata/SystemMetadataServiceTestBase.java index 8643855162fa7..7067dd3a6763e 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/systemmetadata/SystemMetadataServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/systemmetadata/SystemMetadataServiceTestBase.java @@ -11,6 +11,7 @@ import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl; import com.linkedin.mxe.SystemMetadata; +import java.util.Collections; import java.util.List; import javax.annotation.Nonnull; import org.opensearch.client.RestHighLevelClient; @@ -38,7 +39,7 @@ public abstract class SystemMetadataServiceTestBase extends AbstractTestNGSpring @BeforeClass public void setup() { _client = buildService(); - _client.configure(); + _client.reindexAll(Collections.emptySet()); } @BeforeMethod diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java index 3ccd085cab9c0..b44f01d90dae4 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java @@ -58,6 +58,7 @@ import io.datahubproject.test.metadata.context.TestOperationContexts; import java.net.URISyntaxException; import java.util.Calendar; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -128,7 +129,7 @@ public void setup() throws RemoteInvocationException, URISyntaxException { entityRegistry, new IndexConventionImpl("es_timeseries_aspect_service_test")); elasticSearchTimeseriesAspectService = buildService(); - elasticSearchTimeseriesAspectService.configure(); + elasticSearchTimeseriesAspectService.reindexAll(Collections.emptySet()); EntitySpec entitySpec = entityRegistry.getEntitySpec(ENTITY_NAME); aspectSpec = entitySpec.getAspectSpec(ASPECT_NAME); } diff --git a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java index 5da970b46afc7..28a4a2b00cd6f 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java @@ -40,6 +40,7 @@ import io.datahubproject.test.metadata.context.TestOperationContexts; import io.datahubproject.test.search.config.SearchCommonTestConfiguration; import java.io.IOException; +import java.util.Collections; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -149,6 +150,7 @@ protected EntityIndexBuilders entityIndexBuildersHelper(OperationContext opConte Map.of(), true, false, + false, new ElasticSearchConfiguration(), gitVersion); SettingsBuilder settingsBuilder = new SettingsBuilder(null); @@ -252,7 +254,7 @@ public SearchService searchServiceHelper( ranker); // Build indices & write fixture data - indexBuilders.reindexAll(); + indexBuilders.reindexAll(Collections.emptySet()); FixtureReader.builder() .bulkProcessor(_bulkProcessor) diff --git a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java index 34598821f43fd..4cd818db34bf4 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java @@ -39,6 +39,7 @@ import io.datahubproject.test.search.config.SearchCommonTestConfiguration; import io.datahubproject.test.search.config.SearchTestContainerConfiguration; import java.io.IOException; +import java.util.Collections; import java.util.Map; import java.util.Optional; import javax.annotation.Nonnull; @@ -100,6 +101,7 @@ protected EntityIndexBuilders entityIndexBuilders( Map.of(), true, false, + false, new ElasticSearchConfiguration(), gitVersion); SettingsBuilder settingsBuilder = new SettingsBuilder(null); @@ -151,6 +153,7 @@ protected ESIndexBuilder esIndexBuilder() { Map.of(), true, true, + false, new ElasticSearchConfiguration(), gitVersion); } @@ -174,7 +177,7 @@ protected ElasticSearchGraphService graphService( indexConvention, GraphQueryConfiguration.testDefaults), indexBuilder); - graphService.configure(); + graphService.reindexAll(Collections.emptySet()); return graphService; } @@ -225,7 +228,7 @@ protected SearchService searchService( ranker); // Build indices - indexBuilders.reindexAll(); + indexBuilders.reindexAll(Collections.emptySet()); return service; } diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java index f1fd371fdf7bd..a71c40b70f2b4 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java +++ b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java @@ -2,6 +2,7 @@ import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.AUTO_COMPLETE_ENTITY_TYPES; import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.SEARCHABLE_ENTITY_TYPES; +import static org.mockito.Mockito.mock; import com.datahub.authentication.Authentication; import com.datahub.plugins.auth.authorization.Authorizer; @@ -13,6 +14,7 @@ import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.SearchableEntityType; import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.graph.LineageDirection; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.search.LineageSearchResult; @@ -183,7 +185,7 @@ public static LineageSearchResult lineage( .collect(Collectors.toList()), "*", hops, - ResolverUtils.buildFilter(filters, List.of()), + ResolverUtils.buildFilter(filters, List.of(), mock(AspectRetriever.class)), null, 0, 100); diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchTestContainerConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchTestContainerConfiguration.java index 0ddfd77399325..ab6644ce6ff6d 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchTestContainerConfiguration.java +++ b/metadata-io/src/test/java/io/datahubproject/test/search/config/SearchTestContainerConfiguration.java @@ -93,6 +93,7 @@ protected ESIndexBuilder getIndexBuilder( Map.of(), false, false, + false, new ElasticSearchConfiguration(), gitVersion); } diff --git a/metadata-jobs/mae-consumer-job/build.gradle b/metadata-jobs/mae-consumer-job/build.gradle index f3d1ca9885044..4980f81d61bb4 100644 --- a/metadata-jobs/mae-consumer-job/build.gradle +++ b/metadata-jobs/mae-consumer-job/build.gradle @@ -30,6 +30,7 @@ dependencies { implementation externalDependency.logbackClassic testImplementation project(':metadata-dao-impl:kafka-producer') + testImplementation project(':metadata-jobs:pe-consumer') testImplementation externalDependency.springBootTest testImplementation externalDependency.mockito testImplementation externalDependency.testng diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java index 35693ec96b3de..9a4c01dabf9a7 100644 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java @@ -15,6 +15,8 @@ @ComponentScan( basePackages = { "com.linkedin.gms.factory.common", + "com.linkedin.metadata.service", + "com.datahub.event", "com.linkedin.gms.factory.kafka", "com.linkedin.gms.factory.kafka.common", "com.linkedin.gms.factory.kafka.schemaregistry", @@ -23,12 +25,15 @@ "com.linkedin.metadata.dao.producer", "com.linkedin.gms.factory.config", "com.linkedin.gms.factory.entity.update.indices", + "com.linkedin.gms.factory.search", "com.linkedin.gms.factory.entityclient", "com.linkedin.gms.factory.form", "com.linkedin.gms.factory.incident", "com.linkedin.gms.factory.timeline.eventgenerator", "io.datahubproject.metadata.jobs.common.health.kafka", - "com.linkedin.gms.factory.context" + "com.linkedin.gms.factory.context", + "com.linkedin.gms.factory.timeseries", + "com.linkedin.gms.factory.assertion", }, excludeFilters = { @ComponentScan.Filter( diff --git a/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTest.java b/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTest.java index 4e413c8598fb4..c9bf25200513f 100644 --- a/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTest.java +++ b/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTest.java @@ -2,6 +2,7 @@ import static org.testng.AssertJUnit.*; +import com.datahub.event.PlatformEventProcessor; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.service.FormService; @@ -14,7 +15,8 @@ @ActiveProfiles("test") @SpringBootTest( - classes = {MaeConsumerApplication.class, MaeConsumerApplicationTestConfiguration.class}) + classes = {MaeConsumerApplication.class, MaeConsumerApplicationTestConfiguration.class}, + properties = "PE_CONSUMER_ENABLED=true") public class MaeConsumerApplicationTest extends AbstractTestNGSpringContextTests { @Autowired private EntityService mockEntityService; @@ -25,10 +27,13 @@ public class MaeConsumerApplicationTest extends AbstractTestNGSpringContextTests @Autowired private EntitySearchService entitySearchService; + @Autowired private PlatformEventProcessor platformEventProcessor; + @Test public void testMaeConsumerAutoWiring() { assertNotNull(mockEntityService); assertNotNull(kafkaHealthIndicator); assertNotNull(formService); + assertNotNull(platformEventProcessor); } } diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/form/FormAssignmentHook.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/form/FormAssignmentHook.java index d06c6f5102dfb..8d093fe0b8a12 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/form/FormAssignmentHook.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/form/FormAssignmentHook.java @@ -38,9 +38,6 @@ *

3. When a form is hard deleted, any automations used for assigning the form, or validating * prompts, are automatically deleted. * - *

Note that currently, Datasets, Dashboards, Charts, Data Jobs, Data Flows, Containers, are the - * only asset types supported for this hook. - * *

TODO: In the future, let's decide whether we want to support automations to auto-mark form * prompts as "completed" when they do in fact have the correct metadata. (Without user needing to * explicitly fill out a form prompt response) diff --git a/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java b/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java index 08ff802c37e40..ba650c25a6117 100644 --- a/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java +++ b/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java @@ -47,7 +47,8 @@ public SystemEntityClient systemEntityClient( new ExponentialBackoff(1), 1, configurationProvider.getCache().getClient().getEntityClient(), - 1); + 1, + 2); } @MockBean public Database ebeanServer; diff --git a/metadata-jobs/mce-consumer/build.gradle b/metadata-jobs/mce-consumer/build.gradle index b062547724138..5ea24059a3ee3 100644 --- a/metadata-jobs/mce-consumer/build.gradle +++ b/metadata-jobs/mce-consumer/build.gradle @@ -22,6 +22,7 @@ dependencies { implementation project(':metadata-events:mxe-utils-avro') implementation project(':metadata-io') implementation project(':metadata-service:restli-client-api') + implementation project(':metadata-dao-impl:kafka-producer') implementation spec.product.pegasus.restliClient implementation spec.product.pegasus.restliCommon implementation externalDependency.elasticSearchRest diff --git a/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/BusinessAttributeUpdateHookTest.java b/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/BusinessAttributeUpdateHookTest.java index 9db3a77e710a3..47740b02d6166 100644 --- a/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/BusinessAttributeUpdateHookTest.java +++ b/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/BusinessAttributeUpdateHookTest.java @@ -76,7 +76,7 @@ public void setupTest() throws URISyntaxException { mockUpdateIndicesService = mock(UpdateIndicesService.class); actorUrn = Urn.createFromString(TEST_ACTOR_URN); businessAttributeServiceHook = - new BusinessAttributeUpdateHookService(mockUpdateIndicesService, 100, 1); + new BusinessAttributeUpdateHookService(mockUpdateIndicesService, 100, 1, 10, 60); businessAttributeUpdateHook = new BusinessAttributeUpdateHook(businessAttributeServiceHook, true); } @@ -153,7 +153,7 @@ private void testMCLOnInvalidCategory() throws Exception { // verify Mockito.verifyNoInteractions(opContext.getRetrieverContext().get().getGraphRetriever()); - Mockito.verifyNoInteractions(opContext.getRetrieverContext().get().getAspectRetriever()); + Mockito.verifyNoInteractions(opContext.getAspectRetrieverOpt().get()); Mockito.verifyNoInteractions(mockUpdateIndicesService); } @@ -232,7 +232,7 @@ private OperationContext mockOperationContextWithGraph(List graph TestOperationContexts.systemContextNoSearchAuthorization(mockRetrieverContext); // reset mock for test - reset(opContext.getRetrieverContext().get().getAspectRetriever()); + reset(opContext.getAspectRetrieverOpt().get()); if (!graphEdges.isEmpty()) { diff --git a/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/CustomDataQualityRulesMCPSideEffect.java b/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/CustomDataQualityRulesMCPSideEffect.java index de9d3419c216e..af70b56f8ec11 100644 --- a/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/CustomDataQualityRulesMCPSideEffect.java +++ b/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/CustomDataQualityRulesMCPSideEffect.java @@ -4,6 +4,8 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.metadata.aspect.RetrieverContext; import com.linkedin.metadata.aspect.batch.ChangeMCP; +import com.linkedin.metadata.aspect.batch.MCLItem; +import com.linkedin.metadata.aspect.batch.MCPItem; import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl; import java.util.Collection; @@ -33,6 +35,12 @@ protected Stream applyMCPSideEffect( }); } + @Override + protected Stream postMCPSideEffect( + Collection collection, @Nonnull RetrieverContext retrieverContext) { + return Stream.empty(); + } + @Nonnull @Override public AspectPluginConfig getConfig() { diff --git a/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/spring/validation/CustomDataQualityRulesValidator.java b/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/spring/validation/CustomDataQualityRulesValidator.java index 3cd1acaf9645d..5181d749ba3fc 100644 --- a/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/spring/validation/CustomDataQualityRulesValidator.java +++ b/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/spring/validation/CustomDataQualityRulesValidator.java @@ -49,8 +49,7 @@ protected Stream validateProposedAspects( DataQualityRules rules = new DataQualityRules(item.getRecordTemplate().data()); // Enforce at least 1 rule return rules.getRules().isEmpty() - ? new AspectValidationException( - item.getUrn(), item.getAspectName(), "At least one rule is required.") + ? AspectValidationException.forItem(item, "At least one rule is required.") : null; }) .filter(Objects::nonNull); @@ -79,9 +78,8 @@ protected Stream validatePreCommitAspects( if (!newFieldTypeMap .getOrDefault(oldRule.getField(), oldRule.getType()) .equals(oldRule.getType())) { - return new AspectValidationException( - changeMCP.getUrn(), - changeMCP.getAspectName(), + return AspectValidationException.forItem( + changeMCP, String.format( "Field type mismatch. Field: %s Old: %s New: %s", oldRule.getField(), diff --git a/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/validation/CustomDataQualityRulesValidator.java b/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/validation/CustomDataQualityRulesValidator.java index b95d3381d9c8f..590e09cd816c5 100644 --- a/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/validation/CustomDataQualityRulesValidator.java +++ b/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/validation/CustomDataQualityRulesValidator.java @@ -26,8 +26,7 @@ protected Stream validateProposedAspects( DataQualityRules rules = new DataQualityRules(item.getRecordTemplate().data()); // Enforce at least 1 rule return rules.getRules().isEmpty() - ? new AspectValidationException( - item.getUrn(), item.getAspectName(), "At least one rule is required.") + ? AspectValidationException.forItem(item, "At least one rule is required.") : null; }) .filter(Objects::nonNull); @@ -56,9 +55,8 @@ protected Stream validatePreCommitAspects( if (!newFieldTypeMap .getOrDefault(oldRule.getField(), oldRule.getType()) .equals(oldRule.getType())) { - return new AspectValidationException( - changeMCP.getUrn(), - changeMCP.getAspectName(), + return AspectValidationException.forItem( + changeMCP, String.format( "Field type mismatch. Field: %s Old: %s New: %s", oldRule.getField(), diff --git a/metadata-models/docs/entities/dataset.md b/metadata-models/docs/entities/dataset.md index 57bbf2194366f..3d460406029ac 100644 --- a/metadata-models/docs/entities/dataset.md +++ b/metadata-models/docs/entities/dataset.md @@ -88,7 +88,7 @@ Here is an example of how you can add a tag to a field in a dataset using the lo

Python SDK: Add a tag to a column (field) of a dataset ```python -{{ inline /metadata-ingestion/examples/library/dataset_add_column_term.py show_path_as_comment }} +{{ inline /metadata-ingestion/examples/library/dataset_add_column_tag.py show_path_as_comment }} ``` diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionInfo.pdl index 5b60aa18e87da..c9af15a8d019a 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionInfo.pdl @@ -2,6 +2,7 @@ namespace com.linkedin.assertion import com.linkedin.common.CustomProperties import com.linkedin.common.ExternalReference +import com.linkedin.common.AuditStamp /** * Information about an assertion @@ -16,7 +17,8 @@ record AssertionInfo includes CustomProperties, ExternalReference { @Searchable = { } type: enum AssertionType { /** - * A single-dataset assertion. When this is the value, the datasetAssertion field will be populated. + * A single-dataset assertion. + * When this is the value, the datasetAssertion field will be populated. */ DATASET @@ -48,6 +50,14 @@ record AssertionInfo includes CustomProperties, ExternalReference { * Would have named this SCHEMA but the codegen for PDL does not allow this (reserved word). */ DATA_SCHEMA + + /** + * A custom assertion. + * When this is the value, the customAssertion field will be populated. + * Use this assertion type when the exact type of assertion is not modeled in DataHub or + * as a starting point when integrating third-party data quality tools. + */ + CUSTOM } /** @@ -70,11 +80,21 @@ record AssertionInfo includes CustomProperties, ExternalReference { */ sqlAssertion: optional SqlAssertionInfo + /** + * A Field Assertion definition. This field is populated when the type is FIELD. + */ + fieldAssertion: optional FieldAssertionInfo + /** * An schema Assertion definition. This field is populated when the type is DATA_SCHEMA */ schemaAssertion: optional SchemaAssertionInfo + /** + * A Custom Assertion definition. This field is populated when type is CUSTOM. + */ + customAssertion: optional CustomAssertionInfo + /** * The source or origin of the Assertion definition. * @@ -83,6 +103,12 @@ record AssertionInfo includes CustomProperties, ExternalReference { */ source: optional AssertionSource + /** + * The time at which the assertion was last updated and the actor who updated it. + * This field is only present for Native assertions updated after this field was introduced. + */ + lastUpdated: optional AuditStamp + /** * An optional human-readable description of the assertion */ diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResultError.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResultError.pdl index e768fe8521942..4bbfa20f8663e 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResultError.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionResultError.pdl @@ -33,6 +33,14 @@ record AssertionResultError { */ UNSUPPORTED_PLATFORM /** + * Error while executing a custom SQL assertion + */ + CUSTOM_SQL_ERROR + /** + * Error while executing a field assertion + */ + FIELD_ASSERTION_ERROR + /** * Unknown error */ UNKNOWN_ERROR @@ -42,4 +50,4 @@ record AssertionResultError { * Additional metadata depending on the type of error */ properties: optional map[string, string] -} \ No newline at end of file +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionSource.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionSource.pdl index d8892c0c71c6f..734a48f771886 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionSource.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionSource.pdl @@ -1,5 +1,7 @@ namespace com.linkedin.assertion +import com.linkedin.common.AuditStamp + /** * The source of an assertion */ @@ -24,4 +26,10 @@ record AssertionSource { */ INFERRED } + + /** + * The time at which the assertion was initially created and the author who created it. + * This field is only present for Native assertions created after this field was introduced. + */ + created: optional AuditStamp } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionStdOperator.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionStdOperator.pdl index 2e0dcbe24986b..ee4f961249025 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionStdOperator.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionStdOperator.pdl @@ -34,6 +34,16 @@ enum AssertionStdOperator { */ EQUAL_TO + /** + * Value being asserted is not equal to value. Requires 'value' parameter. + */ + NOT_EQUAL_TO + + /** + * Value being asserted is null. Requires no parameters. + */ + NULL + /** * Value being asserted is not null. Requires no parameters. */ @@ -69,6 +79,16 @@ enum AssertionStdOperator { */ NOT_IN + /** + * Value being asserted is true. Requires no parameters. + */ + IS_TRUE + + /** + * Value being asserted is false. Requires no parameters. + */ + IS_FALSE + /** * Other */ diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionStdParameter.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionStdParameter.pdl index a212fe84aff13..9c3e3ea7c1c95 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionStdParameter.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/AssertionStdParameter.pdl @@ -13,10 +13,29 @@ record AssertionStdParameter { * The type of the parameter */ type: enum AssertionStdParameterType { + /** + * A string value + */ STRING + + /** + * A numeric value + */ NUMBER + + /** + * A list of values. When used, value should be formatted as a serialized JSON array. + */ LIST + + /** + * A set of values. When used, value should be formatted as a serialized JSON array. + */ SET + + /** + * A value of unknown type + */ UNKNOWN } } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/CustomAssertionInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/CustomAssertionInfo.pdl new file mode 100644 index 0000000000000..9473b2b3c22bf --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/CustomAssertionInfo.pdl @@ -0,0 +1,43 @@ +namespace com.linkedin.assertion + +import com.linkedin.common.Urn + +/** +* Attributes that are applicable to Custom Assertions +**/ +record CustomAssertionInfo { + /** + * The type of custom assertion. + * This is how your assertion will appear categorized in DataHub UI. + */ + @Searchable = { + "fieldName": "customType" + } + type: string + + /** + * The entity targeted by this assertion. + * This can have support more entityTypes (e.g. dataJob) in future + */ + @Relationship = { + "name": "Asserts", + "entityTypes": [ "dataset" ] + } + entity: Urn + + /** + * dataset schema field targeted by this assertion. + * + * This field is expected to be provided if the assertion is on dataset field + */ + @Relationship = { + "name": "Asserts", + "entityTypes": [ "schemaField" ] + } + field: optional Urn + + /* + * Logic for the assertion as expressed in the native assertion language. Code fragments, query strings, etc. + */ + logic: optional string +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/FieldAssertionInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/FieldAssertionInfo.pdl new file mode 100644 index 0000000000000..0b8d9ab8cceb8 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/FieldAssertionInfo.pdl @@ -0,0 +1,57 @@ +namespace com.linkedin.assertion + +import com.linkedin.common.Urn +import com.linkedin.dataset.DatasetFilter + +/** +* Attributes defining a Field Assertion. +**/ +record FieldAssertionInfo { + /** + * The type of the field assertion being monitored. + */ + @Searchable = {} + type: enum FieldAssertionType { + /** + * An assertion used to validate the values contained with a field / column given a set of rows. + */ + FIELD_VALUES + /** + * An assertion used to validate the value of a common field / column metric (e.g. aggregation) such as null count + percentage, + * min, max, median, and more. + */ + FIELD_METRIC + } + + /** + * The entity targeted by this Field check. + */ + @Searchable = { + "fieldType": "URN" + } + @Relationship = { + "name": "Asserts", + "entityTypes": [ "dataset" ] + } + entity: Urn + + /** + * The definition of an assertion that validates individual values of a field / column for a set of rows. + * This type of assertion verifies that each column value meets a particular requirement. + */ + fieldValuesAssertion: optional FieldValuesAssertion + + /** + * The definition of an assertion that validates a common metric obtained about a field / column for a set of rows. + * This type of assertion verifies that the value of a high-level metric obtained by aggregating over a column meets + * expectations + */ + fieldMetricAssertion: optional FieldMetricAssertion + + /** + * A definition of the specific filters that should be applied, when performing monitoring. + * If not provided, there is no filter, and the full table is under consideration. + * If using DataHub Dataset Profiles as the assertion source type, the value of this field will be ignored. + */ + filter: optional DatasetFilter +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/FieldMetricAssertion.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/FieldMetricAssertion.pdl new file mode 100644 index 0000000000000..ca9ce9cbd6a8c --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/FieldMetricAssertion.pdl @@ -0,0 +1,39 @@ +namespace com.linkedin.assertion + +import com.linkedin.schema.SchemaFieldSpec + +/** +* Attributes defining a field metric assertion, which asserts an expectation against +* a common metric derived from the set of field / column values, for example: +* max, min, median, null count, null percentage, unique count, unique percentage, and more. +*/ +record FieldMetricAssertion { + /** + * The field under evaluation + */ + @Searchable = { + "/path": { + "fieldName": "fieldPath" + } + } + field: SchemaFieldSpec + + /** + * The specific metric to assert against. This is the value that + * will be obtained by applying a standard operation, such as an aggregation, + * to the selected field. + */ + metric: FieldMetricType + + /** + * The predicate to evaluate against the metric for the field / column. + * Depending on the operator, parameters may be required in order to successfully + * evaluate the assertion against the metric value. + */ + operator: AssertionStdOperator + + /** + * Standard parameters required for the assertion. e.g. min_value, max_value, value, columns + */ + parameters: optional AssertionStdParameters +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/FieldMetricType.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/FieldMetricType.pdl new file mode 100644 index 0000000000000..9df06e9dc1fe2 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/FieldMetricType.pdl @@ -0,0 +1,94 @@ +namespace com.linkedin.assertion + +/** + * A standard metric that can be derived from the set of values + * for a specific field / column of a dataset / table. + */ +enum FieldMetricType { + /** + * The number of unique values found in the column value set + */ + UNIQUE_COUNT + + /** + * The percentage of unique values to total rows for the dataset + */ + UNIQUE_PERCENTAGE + + /** + * The number of null values found in the column value set + */ + NULL_COUNT + + /** + * The percentage of null values to total rows for the dataset + */ + NULL_PERCENTAGE + + /** + * The minimum value in the column set (applies to numeric columns) + */ + MIN + + /** + * The maximum value in the column set (applies to numeric columns) + */ + MAX + + /** + * The mean length found in the column set (applies to numeric columns) + */ + MEAN + + /** + * The median length found in the column set (applies to numeric columns) + */ + MEDIAN + + /** + * The stddev length found in the column set (applies to numeric columns) + */ + STDDEV + + /** + * The number of negative values found in the value set (applies to numeric columns) + */ + NEGATIVE_COUNT + + /** + * The percentage of negative values to total rows for the dataset (applies to numeric columns) + */ + NEGATIVE_PERCENTAGE + + /** + * The number of zero values found in the value set (applies to numeric columns) + */ + ZERO_COUNT + + /** + * The percentage of zero values to total rows for the dataset (applies to numeric columns) + */ + ZERO_PERCENTAGE + + /** + * The minimum length found in the column set (applies to string columns) + */ + MIN_LENGTH + + /** + * The maximum length found in the column set (applies to string columns) + */ + MAX_LENGTH + + /** + * The number of empty string values found in the value set (applies to string columns). + * Note: This is a completely different metric different from NULL_COUNT! + */ + EMPTY_COUNT + + /** + * The percentage of empty string values to total rows for the dataset (applies to string columns) + * Note: This is a completely different metric different from NULL_PERCENTAGE! + */ + EMPTY_PERCENTAGE +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/FieldTransform.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/FieldTransform.pdl new file mode 100644 index 0000000000000..3b3d3339a9b86 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/FieldTransform.pdl @@ -0,0 +1,21 @@ +namespace com.linkedin.assertion + +/** +* Definition of a transform applied to the values of a column / field. +* Note that the applicability of a field transform ultimately depends on the native type +* of the field / column. +* +* Model has single field to permit extension. +*/ +record FieldTransform { + /** + * The type of the field transform, e.g. the transformation + * function / operator to apply. + */ + type: enum FieldTransformType { + /** + * Obtain the length of a string field / column (applicable to string types) + */ + LENGTH + } +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/FieldValuesAssertion.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/FieldValuesAssertion.pdl new file mode 100644 index 0000000000000..0400124234462 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/FieldValuesAssertion.pdl @@ -0,0 +1,83 @@ +namespace com.linkedin.assertion + +import com.linkedin.schema.SchemaFieldSpec + +/** +* Attributes defining a field values assertion, which asserts that the values for a field / column +* of a dataset / table matches a set of expectations. +* +* In other words, this type of assertion acts as a semantic constraint applied to fields for a specific column. +* +* TODO: We should display the "failed row count" to the user if the column fails the verification rules. +* TODO: Determine whether we need an "operator" that can be applied to the field. +*/ +record FieldValuesAssertion { + /** + * The field under evaluation + */ + @Searchable = { + "/path": { + "fieldName": "fieldPath" + } + } + field: SchemaFieldSpec + + /** + * An optional transform to apply to field values + * before evaluating the operator. + * + * If none is applied, the field value will be compared as is. + */ + transform: optional FieldTransform + + /** + * The predicate to evaluate against a single value of the field. + * Depending on the operator, parameters may be required in order to successfully + * evaluate the assertion against the field value. + */ + operator: AssertionStdOperator + + /** + * Standard parameters required for the assertion. e.g. min_value, max_value, value, columns + */ + parameters: optional AssertionStdParameters + + /** + * Additional customization about when the assertion + * should be officially considered failing. + */ + failThreshold: record FieldValuesFailThreshold { + + /** + * The type of failure threshold. Either based on the number + * of column values (rows) that fail the expectations, or the percentage + * of the total rows under consideration. + */ + type: enum FieldValuesFailThresholdType { + /* + * The maximum number of column values (i.e. rows) that are allowed + * to fail the defined expectations before the assertion officially fails. + */ + COUNT + /* + * The maximum percentage of rows that are allowed + * to fail the defined column expectations before the assertion officially fails. + */ + PERCENTAGE + } = "COUNT" + + /** + * By default this is 0, meaning that ALL column values (i.e. rows) must + * meet the defined expectations. + */ + value: long = 0 + } + + /** + * Whether to ignore or allow nulls when running the values assertion. (i.e. + * consider only non-null values) using operators OTHER than the IS_NULL operator. + * + * Defaults to true, allowing null values. + */ + excludeNulls: boolean = true +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessAssertionSchedule.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessAssertionSchedule.pdl index a87342ad4f5ed..1905cb114e08c 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessAssertionSchedule.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessAssertionSchedule.pdl @@ -15,7 +15,7 @@ record FreshnessAssertionSchedule { */ type: enum FreshnessAssertionScheduleType { /** - * An highly configurable recurring schedule which describes the times of events described + * A highly configurable recurring schedule which describes the times of events described * by a CRON schedule, with the evaluation schedule assuming to be matching the cron schedule. * * In a CRON schedule type, we compute the look-back window to be the time between the last scheduled event @@ -45,12 +45,31 @@ record FreshnessAssertionSchedule { * to be evaluated each hour, we'd compute the result as follows: * * 1. Subtract the fixed interval from the current time (Evaluation time) to compute the bounds of a fixed look-back window. - * 2. Verify that the target event has occurred within the CRON-interval window. + * 2. Verify that the target event has occurred within the look-back window. * 3. If the target event has occurred within the time window, then assertion passes. * 4. If the target event has not occurred within the time window, then the assertion fails. * */ FIXED_INTERVAL + /** + * A stateful check that takes the last time this check ran to determine the look-back window. + * + * To compute the valid look-back- window, we start at the time the monitor last evaluated this assertion, + * and we end at the point in time the check is currently running. + * + * For example, let's say a Freshness assertion is of type SINCE_THE_LAST_CHECK, and the monitor is configured to + * run every day at 12:00am. Let's assume this assertion was last evaluated yesterday at 12:04am. We'd compute + * the result as follows: + * + * 1. Get the timestamp for the last run of the monitor on this assertion. + * 2. look_back_window_start_time = latest_monitor_run.timestampMillis [ie. 12:04a yesterday] + * 3. look_back_window_end_time = nowMillis [ie. 12:02a today] + * 4. If the target event has occurred within the window [ie. 12:04a yday to 12:02a today], + * then the assertion passes. + * 5. If the target event has not occurred within the window, then the assertion fails. + * + */ + SINCE_THE_LAST_CHECK } /** diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessFieldSpec.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessFieldSpec.pdl index 04acd1c71352d..179d4a1b13591 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessFieldSpec.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/FreshnessFieldSpec.pdl @@ -4,11 +4,13 @@ import com.linkedin.schema.SchemaFieldSpec /** -* Lightweight spec used for referencing a particular schema field. -**/ +* Lightweight spec used for referencing a particular schema field that is used to compute +* a freshness signal or operation. +* TODO: Since this is now leveraged across assertions & metrics / operations, we should consider moving this to a common package. +*/ record FreshnessFieldSpec includes SchemaFieldSpec { /** - * The type of the field being used to verify the Freshness Assertion. + * The type of the field being used to verify the Freshness of the asset. */ kind: optional FreshnessFieldKind } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/SchemaAssertionInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/SchemaAssertionInfo.pdl index fd246e0c7cfc4..2e691d5152ae3 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/assertion/SchemaAssertionInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/SchemaAssertionInfo.pdl @@ -25,5 +25,36 @@ record SchemaAssertionInfo { * Note that many of the fields of this model, especially those related to metadata (tags, terms) * will go unused in this context. */ - schema: SchemaMetadata +// @Relationship = { +// "/foreignKeys/*/foreignFields/*": null, +// "/foreignKeys/*/foreignDataset": null, +// "/fields/*/globalTags/tags/*/tag": null, +// "/fields/*/glossaryTerms/terms/*/urn": null +// } +// @Searchable = { +// "/fields/*/fieldPath": null, +// "/fields/*/description": null, +// "/fields/*/label": null, +// "/fields/*/globalTags/tags/*/tag": null, +// "/fields/*/glossaryTerms/terms/*/urn": null +// } + schema: SchemaMetadata + + /** + * The required compatibility level for the schema assertion to pass. + */ + compatibility: optional enum SchemaAssertionCompatibility { + /** + * The actual schema must be exactly the same as the expected schema + */ + EXACT_MATCH, + /** + * The actual schema must be a superset of the expected schema + */ + SUPERSET, + /** + * The actual schema must be a subset of the expected schema + */ + SUBSET + } = "EXACT_MATCH" } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/assertion/VolumeAssertionInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/assertion/VolumeAssertionInfo.pdl index 327b76f95762e..bdc78d3bd0a6f 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/assertion/VolumeAssertionInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/assertion/VolumeAssertionInfo.pdl @@ -8,7 +8,7 @@ import com.linkedin.dataset.DatasetFilter */ record VolumeAssertionInfo { /** - * The type of the freshness assertion being monitored. + * The type of the volume assertion being monitored. */ @Searchable = {} type: enum VolumeAssertionType { diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/CronSchedule.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/CronSchedule.pdl new file mode 100644 index 0000000000000..e59ef345186be --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/CronSchedule.pdl @@ -0,0 +1,16 @@ +namespace com.linkedin.common + +/** +* Attributes defining a CRON-formatted schedule. +*/ +record CronSchedule { + /** + * A cron-formatted execution interval, as a cron string, e.g. 1 * * * * + */ + cron: string + + /** + * Timezone in which the cron interval applies, e.g. America/Los Angeles + */ + timezone: string +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/datacontract/DataQualityContract.pdl b/metadata-models/src/main/pegasus/com/linkedin/datacontract/DataQualityContract.pdl index 273d2c2a56f95..3ff8b58284f18 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/datacontract/DataQualityContract.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/datacontract/DataQualityContract.pdl @@ -12,5 +12,9 @@ record DataQualityContract { * The assertion representing the Data Quality contract. * E.g. a table or column-level assertion. */ + @Relationship = { + "name": "IncludesDataQualityAssertion", + "entityTypes": [ "assertion" ] + } assertion: Urn } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/datacontract/SchemaContract.pdl b/metadata-models/src/main/pegasus/com/linkedin/datacontract/SchemaContract.pdl index 6c11e0da5b128..af61a660cdf76 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/datacontract/SchemaContract.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/datacontract/SchemaContract.pdl @@ -9,5 +9,9 @@ record SchemaContract { /** * The assertion representing the schema contract. */ + @Relationship = { + "name": "IncludesSchemaAssertion", + "entityTypes": [ "assertion" ] + } assertion: Urn } diff --git a/metadata-models/src/main/pegasus/com/linkedin/domain/DomainProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/domain/DomainProperties.pdl index 89f44a433b7ba..eb307b726855d 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/domain/DomainProperties.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/domain/DomainProperties.pdl @@ -1,6 +1,7 @@ namespace com.linkedin.domain import com.linkedin.common.AuditStamp +import com.linkedin.common.CustomProperties import com.linkedin.common.Urn /** @@ -9,7 +10,7 @@ import com.linkedin.common.Urn @Aspect = { "name": "domainProperties" } -record DomainProperties { +record DomainProperties includes CustomProperties { /** * Display name of the Domain diff --git a/metadata-models/src/main/pegasus/com/linkedin/execution/ExecutionRequestResult.pdl b/metadata-models/src/main/pegasus/com/linkedin/execution/ExecutionRequestResult.pdl index 29acd0aa52389..606c3a06bc74b 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/execution/ExecutionRequestResult.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/execution/ExecutionRequestResult.pdl @@ -10,6 +10,10 @@ record ExecutionRequestResult { /** * The status of the execution request */ + @Searchable = { + "fieldType": "KEYWORD", + "fieldName": "executionResultStatus" + } status: string /** @@ -36,4 +40,4 @@ record ExecutionRequestResult { * Duration in milliseconds */ durationMs: optional long -} \ No newline at end of file +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserEditableInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserEditableInfo.pdl index 48ee53377e582..9667c93c8b770 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserEditableInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserEditableInfo.pdl @@ -1,6 +1,7 @@ namespace com.linkedin.identity import com.linkedin.common.Url +import com.linkedin.common.Urn /** * Linkedin corp user information that can be edited from UI @@ -56,6 +57,26 @@ record CorpUserEditableInfo { */ title: optional string + /** + * The platforms that the user commonly works with + */ + @Relationship = { + "/*": { + "name": "IsUserOf", + "entityTypes": ["dataPlatform"] + } + } + platforms: optional array[Urn] + + /** + * The user's persona type, based on their role + */ + @Relationship = { + "name": "IsPersona", + "entityTypes": ["dataHubPersona"] + } + persona: optional Urn + /** * Slack handle for the user */ diff --git a/metadata-models/src/main/pegasus/com/linkedin/incident/IncidentSource.pdl b/metadata-models/src/main/pegasus/com/linkedin/incident/IncidentSource.pdl index 2f8912da5458c..2e65d37dc0939 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/incident/IncidentSource.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/incident/IncidentSource.pdl @@ -22,6 +22,11 @@ record IncidentSource { * Manually created incident, via UI or API. */ MANUAL + + /** + * An assertion has failed, triggering the incident. + */ + ASSERTION_FAILURE } /** diff --git a/metadata-models/src/main/pegasus/com/linkedin/incident/IncidentType.pdl b/metadata-models/src/main/pegasus/com/linkedin/incident/IncidentType.pdl index 27c4790e3b6ef..1c3473018d4e0 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/incident/IncidentType.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/incident/IncidentType.pdl @@ -4,6 +4,36 @@ namespace com.linkedin.incident * A type of asset incident */ enum IncidentType { + /** + * An Freshness Assertion has failed, triggering the incident. + * Raised on entities where assertions are configured to generate incidents. + */ + FRESHNESS + + /** + * An Volume Assertion has failed, triggering the incident. + * Raised on entities where assertions are configured to generate incidents. + */ + VOLUME + + /** + * A Field Assertion has failed, triggering the incident. + * Raised on entities where assertions are configured to generate incidents. + */ + FIELD + + /** + * A raw SQL-statement based assertion has failed, triggering the incident. + * Raised on entities where assertions are configured to generate incidents. + */ + SQL + + /** + * A Data Schema assertion has failed, triggering the incident. + * Raised on entities where assertions are configured to generate incidents. + */ + DATA_SCHEMA + /** * A misc. operational incident, e.g. failure to materialize a dataset. */ diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/LineageRelationship.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/LineageRelationship.pdl index 552dd7323b551..7535c7e9292ec 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/LineageRelationship.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/LineageRelationship.pdl @@ -73,6 +73,11 @@ record LineageRelationship { */ explored: optional boolean + /** + * Indicates this destination node has additional unexplored child relationships + */ + truncatedChildren: optional boolean + /** * Whether this relationship was ignored as a hop while performing the graph walk */ diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/DataHubPersonaKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/DataHubPersonaKey.pdl new file mode 100644 index 0000000000000..296444221af3a --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/DataHubPersonaKey.pdl @@ -0,0 +1,14 @@ +namespace com.linkedin.metadata.key + +/** + * Key for a persona type + */ +@Aspect = { + "name": "dataHubPersonaKey" +} +record DataHubPersonaKey { + /** + * A unique id for the persona type + */ + id: string +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/search/LineageSearchEntity.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/search/LineageSearchEntity.pdl index 3fd8a48c6bf5e..3f246b5014df0 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/search/LineageSearchEntity.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/search/LineageSearchEntity.pdl @@ -35,6 +35,11 @@ record LineageSearchEntity includes SearchEntity { */ explored: optional boolean + /** + * Indicates this destination node has additional unexplored child relationships + */ + truncatedChildren: optional boolean + /** * Whether this relationship was ignored as a hop while performing the graph walk */ diff --git a/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLFeatureProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLFeatureProperties.pdl index 9c5ad22b37a0f..71d7ef312cf36 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLFeatureProperties.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLFeatureProperties.pdl @@ -1,5 +1,6 @@ namespace com.linkedin.ml.metadata +import com.linkedin.common.CustomProperties import com.linkedin.common.Urn import com.linkedin.common.MLFeatureDataType import com.linkedin.common.VersionTag @@ -10,7 +11,7 @@ import com.linkedin.common.VersionTag @Aspect = { "name": "mlFeatureProperties" } -record MLFeatureProperties { +record MLFeatureProperties includes CustomProperties { /** * Documentation of the MLFeature diff --git a/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLPrimaryKeyProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLPrimaryKeyProperties.pdl index 4c17737dae302..54f98a5a6f96f 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLPrimaryKeyProperties.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLPrimaryKeyProperties.pdl @@ -1,5 +1,6 @@ namespace com.linkedin.ml.metadata +import com.linkedin.common.CustomProperties import com.linkedin.common.Urn import com.linkedin.common.MLFeatureDataType import com.linkedin.common.VersionTag @@ -10,7 +11,7 @@ import com.linkedin.common.VersionTag @Aspect = { "name": "mlPrimaryKeyProperties" } -record MLPrimaryKeyProperties { +record MLPrimaryKeyProperties includes CustomProperties { /** * Documentation of the MLPrimaryKey diff --git a/metadata-models/src/main/pegasus/com/linkedin/mxe/MetadataChangeProposal.pdl b/metadata-models/src/main/pegasus/com/linkedin/mxe/MetadataChangeProposal.pdl index 3d540f58a686d..f4b3bc2a292d3 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/mxe/MetadataChangeProposal.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/mxe/MetadataChangeProposal.pdl @@ -47,8 +47,12 @@ record MetadataChangeProposal { aspect: optional GenericAspect /** - * A string->string map of custom properties that one might want to attach to an event + * System properties that one might want to attach to an event **/ systemMetadata: optional SystemMetadata + /** + * Headers - intended to mimic http headers + */ + headers: optional map[string, string] } diff --git a/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl b/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl index 101557fca7949..94c8aa77aa5d0 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl @@ -38,4 +38,11 @@ record SystemMetadata { * Additional properties */ properties: optional map[string, string] + + /** + * Aspect version + * Initial implementation will use the aspect version's number, however stored as + * a string in the case where a different aspect versioning scheme is later adopted. + */ + version: optional string } diff --git a/metadata-models/src/main/pegasus/com/linkedin/persona/DataHubPersonaInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/persona/DataHubPersonaInfo.pdl new file mode 100644 index 0000000000000..fb422993b27dd --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/persona/DataHubPersonaInfo.pdl @@ -0,0 +1,10 @@ +namespace com.linkedin.persona + +/** + * Placeholder aspect for persona type info + */ +@Aspect = { + "name": "dataHubPersonaInfo" +} +record DataHubPersonaInfo { +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/query/QueryUsageStatistics.pdl b/metadata-models/src/main/pegasus/com/linkedin/query/QueryUsageStatistics.pdl new file mode 100644 index 0000000000000..275077e5e6a47 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/query/QueryUsageStatistics.pdl @@ -0,0 +1,43 @@ +namespace com.linkedin.query + +import com.linkedin.timeseries.TimeseriesAspectBase +import com.linkedin.dataset.DatasetUserUsageCounts + +/** + * Stats corresponding to dataset's usage. + */ +@Aspect = { + "name": "queryUsageStatistics", + "type": "timeseries", +} +record QueryUsageStatistics includes TimeseriesAspectBase { + /** + * Total query count in this bucket + */ + @TimeseriesField = {} + queryCount: optional int + + /** + * Query cost for this query and bucket + */ + @TimeseriesField = {} + queryCost: optional double + + /** + * Last executed timestamp + */ + @TimeseriesField = {} + lastExecutedAt: optional long + + /** + * Unique user count + */ + @TimeseriesField = {} + uniqueUserCount: optional int + + /** + * Users within this bucket, with frequency counts + */ + @TimeseriesFieldCollection = {"key":"user"} + userCounts: optional array[DatasetUserUsageCounts] +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalViewsSettings.pdl b/metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalViewsSettings.pdl index ff34dea556855..10b1176b637ef 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalViewsSettings.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalViewsSettings.pdl @@ -9,5 +9,9 @@ record GlobalViewsSettings { /** * The default View for the instance, or organization. */ + @Relationship = { + "name": "viewedWith", + "entityTypes": [ "dataHubView" ] + } defaultView: optional Urn -} \ No newline at end of file +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl index 178d7b3cf4376..bf0bf65099b2e 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl @@ -78,5 +78,13 @@ record StructuredPropertyDefinition { "fieldType": "BOOLEAN" } immutable: boolean = false + + /** + * Definition version - Allows breaking schema changes. String is compared case-insensitive and new + * versions must be monotonically increasing. Cannot use periods/dots. + * Suggestions: v1, v2 + * 20240610, 20240611 + */ + version: optional string } diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index 1e8044e3b5f86..c8344b7de1e12 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -110,6 +110,7 @@ entities: - dataProcessInstanceProperties - dataProcessInstanceRelationships - dataProcessInstanceRunEvent + - status - name: chart category: core keyAspect: chartKey @@ -503,6 +504,7 @@ entities: aspects: - queryProperties - querySubjects + - queryUsageStatistics - status - dataPlatformInstance - subTypes @@ -541,6 +543,11 @@ entities: - dataContractProperties - dataContractStatus - status + - name: dataHubPersona + category: internal + keyAspect: dataHubPersonaKey + aspects: + - dataHubPersonaInfo - name: entityType doc: A type of entity in the DataHub Metadata Model. category: core @@ -581,17 +588,22 @@ entities: events: plugins: aspectPayloadValidators: - - className: 'com.linkedin.metadata.aspect.validation.PropertyDefinitionValidator' + - className: 'com.linkedin.metadata.structuredproperties.validation.PropertyDefinitionValidator' + packageScan: + - 'com.linkedin.metadata.structuredproperties.validation' enabled: true supportedOperations: + - CREATE + - CREATE_ENTITY - UPSERT - - DELETE supportedEntityAspectNames: - entityName: structuredProperty aspectName: propertyDefinition - entityName: structuredProperty aspectName: structuredPropertyKey - - className: 'com.linkedin.metadata.aspect.validation.StructuredPropertiesValidator' + - className: 'com.linkedin.metadata.structuredproperties.validation.StructuredPropertiesValidator' + packageScan: + - 'com.linkedin.metadata.structuredproperties.validation' enabled: true supportedOperations: - CREATE @@ -608,8 +620,34 @@ plugins: supportedEntityAspectNames: - entityName: '*' aspectName: '*' + - className: 'com.linkedin.metadata.aspect.validation.ConditionalWriteValidator' + enabled: true + supportedOperations: + - CREATE + - CREATE_ENTITY + - DELETE + - UPSERT + - UPDATE + - PATCH + supportedEntityAspectNames: + - entityName: '*' + aspectName: '*' + mcpSideEffects: + - className: 'com.linkedin.metadata.structuredproperties.hooks.PropertyDefinitionDeleteSideEffect' + packageScan: + - 'com.linkedin.metadata.structuredproperties.hooks' + enabled: true + supportedOperations: + - DELETE + supportedEntityAspectNames: + - entityName: structuredProperty + aspectName: propertyDefinition + - entityName: structuredProperty + aspectName: structuredPropertyKey mutationHooks: - - className: 'com.linkedin.metadata.aspect.hooks.StructuredPropertiesSoftDelete' + - className: 'com.linkedin.metadata.structuredproperties.hooks.StructuredPropertiesSoftDelete' + packageScan: + - 'com.linkedin.metadata.structuredproperties.hooks' enabled: true supportedEntityAspectNames: - entityName: '*' diff --git a/metadata-operation-context/build.gradle b/metadata-operation-context/build.gradle index 1be98cb0140f3..650082ef0d25e 100644 --- a/metadata-operation-context/build.gradle +++ b/metadata-operation-context/build.gradle @@ -7,6 +7,8 @@ dependencies { api project(':metadata-auth:auth-api') implementation externalDependency.slf4jApi + implementation externalDependency.servletApi + implementation spec.product.pegasus.restliServer compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok diff --git a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java index 56247d61337e8..9928318268a3e 100644 --- a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java +++ b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java @@ -7,6 +7,7 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.query.LineageFlags; import com.linkedin.metadata.query.SearchFlags; @@ -285,6 +286,15 @@ public Optional getRetrieverContext() { return Optional.ofNullable(retrieverContext); } + @Nullable + public AspectRetriever getAspectRetriever() { + return getAspectRetrieverOpt().orElse(null); + } + + public Optional getAspectRetrieverOpt() { + return getRetrieverContext().map(RetrieverContext::getAspectRetriever); + } + /** * Return a unique id for this context. Typically useful for building cache keys. We combine the * different context components to create a single string representation of the hashcode across diff --git a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/RequestContext.java b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/RequestContext.java index 83090b9235701..dcea185fcbc7c 100644 --- a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/RequestContext.java +++ b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/RequestContext.java @@ -1,5 +1,8 @@ package io.datahubproject.metadata.context; +import com.google.common.net.HttpHeaders; +import com.linkedin.restli.server.ResourceContext; +import jakarta.servlet.http.HttpServletRequest; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -11,7 +14,9 @@ import javax.annotation.Nullable; import lombok.Builder; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +@Slf4j @Getter @Builder public class RequestContext implements ContextInterface { @@ -19,6 +24,8 @@ public class RequestContext implements ContextInterface { public static final RequestContext TEST = RequestContext.builder().requestID("test").requestAPI(RequestAPI.TEST).build(); + @Nonnull private final String actorUrn; + @Nonnull private final String sourceIP; @Nonnull private final RequestAPI requestAPI; /** @@ -27,6 +34,23 @@ public class RequestContext implements ContextInterface { */ @Nonnull private final String requestID; + @Nonnull private final String userAgent; + + public RequestContext( + @Nonnull String actorUrn, + @Nonnull String sourceIP, + @Nonnull RequestAPI requestAPI, + @Nonnull String requestID, + @Nonnull String userAgent) { + this.actorUrn = actorUrn; + this.sourceIP = sourceIP; + this.requestAPI = requestAPI; + this.requestID = requestID; + this.userAgent = userAgent; + // Uniform common logging of requests across APIs + log.info(toString()); + } + @Override public Optional getCacheKeyComponent() { return Optional.empty(); @@ -34,39 +58,76 @@ public Optional getCacheKeyComponent() { public static class RequestContextBuilder { private RequestContext build() { - return new RequestContext(this.requestAPI, this.requestID); + return new RequestContext( + this.actorUrn, this.sourceIP, this.requestAPI, this.requestID, this.userAgent); } - public RequestContext buildGraphql(@Nonnull String queryName, Map variables) { + public RequestContext buildGraphql( + @Nonnull String actorUrn, + @Nonnull HttpServletRequest request, + @Nonnull String queryName, + Map variables) { + actorUrn(actorUrn); + sourceIP(extractSourceIP(request)); requestAPI(RequestAPI.GRAPHQL); requestID(buildRequestId(queryName, Set.of())); + userAgent(extractUserAgent(request)); return build(); } - public RequestContext buildRestli(String action, @Nullable String entityName) { - return buildRestli(action, entityName == null ? null : List.of(entityName)); + public RequestContext buildRestli( + @Nonnull String actorUrn, + @Nullable ResourceContext resourceContext, + String action, + @Nullable String entityName) { + return buildRestli( + actorUrn, resourceContext, action, entityName == null ? null : List.of(entityName)); } - public RequestContext buildRestli(@Nonnull String action, @Nullable String[] entityNames) { + public RequestContext buildRestli( + @Nonnull String actorUrn, + @Nullable ResourceContext resourceContext, + @Nonnull String action, + @Nullable String[] entityNames) { return buildRestli( + actorUrn, + resourceContext, action, entityNames == null ? null : Arrays.stream(entityNames).collect(Collectors.toList())); } - public RequestContext buildRestli(String action, @Nullable Collection entityNames) { + public RequestContext buildRestli( + @Nonnull String actorUrn, + @Nullable ResourceContext resourceContext, + String action, + @Nullable Collection entityNames) { + actorUrn(actorUrn); + sourceIP(resourceContext == null ? "" : extractSourceIP(resourceContext)); requestAPI(RequestAPI.RESTLI); requestID(buildRequestId(action, entityNames)); + userAgent(resourceContext == null ? "" : extractUserAgent(resourceContext)); return build(); } - public RequestContext buildOpenapi(@Nonnull String action, @Nullable String entityName) { - return buildOpenapi(action, entityName == null ? null : List.of(entityName)); + public RequestContext buildOpenapi( + @Nonnull String actorUrn, + @Nonnull HttpServletRequest request, + @Nonnull String action, + @Nullable String entityName) { + return buildOpenapi( + actorUrn, request, action, entityName == null ? null : List.of(entityName)); } public RequestContext buildOpenapi( - @Nonnull String action, @Nullable Collection entityNames) { + @Nonnull String actorUrn, + @Nullable HttpServletRequest request, + @Nonnull String action, + @Nullable Collection entityNames) { + actorUrn(actorUrn); + sourceIP(request == null ? "" : extractSourceIP(request)); requestAPI(RequestAPI.OPENAPI); requestID(buildRequestId(action, entityNames)); + userAgent(request == null ? "" : extractUserAgent(request)); return build(); } @@ -77,6 +138,46 @@ private static String buildRequestId( : String.format( "%s(%s)", action, entityNames.stream().distinct().collect(Collectors.toList())); } + + private static String extractUserAgent(@Nonnull HttpServletRequest request) { + return Optional.ofNullable(request.getHeader(HttpHeaders.USER_AGENT)).orElse(""); + } + + private static String extractUserAgent(@Nonnull ResourceContext resourceContext) { + return Optional.ofNullable(resourceContext.getRequestHeaders().get(HttpHeaders.USER_AGENT)) + .orElse(""); + } + + private static String extractSourceIP(@Nonnull HttpServletRequest request) { + return Optional.ofNullable(request.getHeader(HttpHeaders.X_FORWARDED_FOR)) + .orElse(request.getRemoteAddr()); + } + + private static String extractSourceIP(@Nonnull ResourceContext resourceContext) { + return Optional.ofNullable( + resourceContext.getRequestHeaders().get(HttpHeaders.X_FORWARDED_FOR)) + .orElse(resourceContext.getRawRequestContext().getLocalAttr("REMOTE_ADDR").toString()); + } + } + + @Override + public String toString() { + return "RequestContext{" + + "actorUrn='" + + actorUrn + + '\'' + + ", sourceIP='" + + sourceIP + + '\'' + + ", requestAPI=" + + requestAPI + + ", requestID='" + + requestID + + '\'' + + ", userAgent='" + + userAgent + + '\'' + + '}'; } public enum RequestAPI { diff --git a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/RetrieverContext.java b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/RetrieverContext.java index 0d7e6359b771c..9337fbfe3bb00 100644 --- a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/RetrieverContext.java +++ b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/RetrieverContext.java @@ -2,6 +2,7 @@ import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.GraphRetriever; +import com.linkedin.metadata.entity.SearchRetriever; import java.util.Optional; import javax.annotation.Nonnull; import lombok.Builder; @@ -14,6 +15,7 @@ public class RetrieverContext @Nonnull private final GraphRetriever graphRetriever; @Nonnull private final AspectRetriever aspectRetriever; + @Nonnull private final SearchRetriever searchRetriever; @Override public Optional getCacheKeyComponent() { diff --git a/metadata-operation-context/src/main/java/io/datahubproject/test/metadata/context/TestOperationContexts.java b/metadata-operation-context/src/main/java/io/datahubproject/test/metadata/context/TestOperationContexts.java index 4c28ba037d3a1..e54c040fe13b5 100644 --- a/metadata-operation-context/src/main/java/io/datahubproject/test/metadata/context/TestOperationContexts.java +++ b/metadata-operation-context/src/main/java/io/datahubproject/test/metadata/context/TestOperationContexts.java @@ -10,7 +10,9 @@ import com.linkedin.entity.Aspect; import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.GraphRetriever; +import com.linkedin.metadata.aspect.SystemAspect; import com.linkedin.metadata.aspect.models.graph.RelatedEntitiesScrollResult; +import com.linkedin.metadata.entity.SearchRetriever; import com.linkedin.metadata.models.registry.ConfigEntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistryException; @@ -19,6 +21,8 @@ import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.RelationshipFilter; import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.search.ScrollResult; +import com.linkedin.metadata.search.SearchEntityArray; import com.linkedin.metadata.snapshot.Snapshot; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl; @@ -55,7 +59,7 @@ public class TestOperationContexts { private static EntityRegistry defaultEntityRegistryInstance; - private static EntityRegistry defaultEntityRegistry() { + public static EntityRegistry defaultEntityRegistry() { if (defaultEntityRegistryInstance == null) { PathSpecBasedSchemaAnnotationVisitor.class .getClassLoader() @@ -84,6 +88,7 @@ public static AspectRetriever emptyAspectRetriever( } public static GraphRetriever emptyGraphRetriever = new EmptyGraphRetriever(); + public static SearchRetriever emptySearchRetriever = new EmptySearchRetriever(); public static RetrieverContext emptyRetrieverContext( @Nullable Supplier entityRegistrySupplier) { @@ -91,6 +96,7 @@ public static RetrieverContext emptyRetrieverContext( return RetrieverContext.builder() .aspectRetriever(emptyAspectRetriever(entityRegistrySupplier)) .graphRetriever(emptyGraphRetriever) + .searchRetriever(emptySearchRetriever) .build(); } @@ -121,6 +127,20 @@ public static OperationContext systemContextNoSearchAuthorization( null); } + public static OperationContext systemContextNoSearchAuthorization( + @Nullable AspectRetriever aspectRetriever) { + RetrieverContext retrieverContext = + RetrieverContext.builder() + .aspectRetriever(aspectRetriever) + .graphRetriever(emptyGraphRetriever) + .searchRetriever(emptySearchRetriever) + .build(); + return systemContextNoSearchAuthorization( + () -> retrieverContext.getAspectRetriever().getEntityRegistry(), + () -> retrieverContext, + null); + } + public static OperationContext systemContextNoSearchAuthorization( @Nullable Supplier retrieverContextSupplier, @Nullable IndexConvention indexConvention) { @@ -250,6 +270,13 @@ public Map> getLatestAspectObjects( return Map.of(); } + @Nonnull + @Override + public Map> getLatestSystemAspects( + Map> urnAspectNames) { + return Map.of(); + } + @Nonnull @Override public EntityRegistry getEntityRegistry() { @@ -277,5 +304,21 @@ public RelatedEntitiesScrollResult scrollRelatedEntities( } } + public static class EmptySearchRetriever implements SearchRetriever { + + @Override + public ScrollResult scroll( + @Nonnull List entities, + @Nullable Filter filters, + @Nullable String scrollId, + int count) { + ScrollResult empty = new ScrollResult(); + empty.setEntities(new SearchEntityArray()); + empty.setNumEntities(0); + empty.setPageSize(0); + return empty; + } + } + private TestOperationContexts() {} } diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authentication/token/StatefulTokenServiceTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authentication/token/StatefulTokenServiceTest.java index bcea100ec733f..d8b7c23153537 100644 --- a/metadata-service/auth-impl/src/test/java/com/datahub/authentication/token/StatefulTokenServiceTest.java +++ b/metadata-service/auth-impl/src/test/java/com/datahub/authentication/token/StatefulTokenServiceTest.java @@ -203,7 +203,8 @@ public void generateRevokeToken() throws TokenException { Mockito.when(mockService.exists(any(OperationContext.class), any(Urn.class), eq(true))) .thenReturn(true); - final RollbackRunResult result = new RollbackRunResult(ImmutableList.of(), 0); + final RollbackRunResult result = + new RollbackRunResult(ImmutableList.of(), 0, ImmutableList.of()); Mockito.when(mockService.deleteUrn(any(OperationContext.class), any(Urn.class))) .thenReturn(result); diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/MetadataChangeProposalConfig.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/MetadataChangeProposalConfig.java new file mode 100644 index 0000000000000..3d3808bc5feb4 --- /dev/null +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/MetadataChangeProposalConfig.java @@ -0,0 +1,30 @@ +package com.linkedin.metadata.config; + +import lombok.Data; +import lombok.experimental.Accessors; + +@Data +@Accessors(chain = true) +public class MetadataChangeProposalConfig { + + ThrottlesConfig throttle; + + @Data + @Accessors(chain = true) + public static class ThrottlesConfig { + Integer updateIntervalMs; + ThrottleConfig versioned; + ThrottleConfig timeseries; + } + + @Data + @Accessors(chain = true) + public static class ThrottleConfig { + boolean enabled; + Integer threshold; + Integer maxAttempts; + Integer initialIntervalMs; + Integer multiplier; + Integer maxIntervalMs; + } +} diff --git a/metadata-service/configuration/src/main/resources/application.yaml b/metadata-service/configuration/src/main/resources/application.yaml index 770be86e254b1..599f7e7be344f 100644 --- a/metadata-service/configuration/src/main/resources/application.yaml +++ b/metadata-service/configuration/src/main/resources/application.yaml @@ -217,9 +217,9 @@ elasticsearch: exactMatch: exclusive: ${ELASTICSEARCH_QUERY_EXACT_MATCH_EXCLUSIVE:false} # if false will only apply weights, if true will exclude non-exact withPrefix: ${ELASTICSEARCH_QUERY_EXACT_MATCH_WITH_PREFIX:true} # include prefix exact matches - exactFactor: ${ELASTICSEARCH_QUERY_EXACT_MATCH_FACTOR:10.0} # boost multiplier when exact with case - prefixFactor: ${ELASTICSEARCH_QUERY_EXACT_MATCH_PREFIX_FACTOR:1.6} # boost multiplier when exact prefix - caseSensitivityFactor: ${ELASTICSEARCH_QUERY_EXACT_MATCH_CASE_FACTOR:0.7} # stacked boost multiplier when case mismatch + exactFactor: ${ELASTICSEARCH_QUERY_EXACT_MATCH_FACTOR:16.0} # boost multiplier when exact with case + prefixFactor: ${ELASTICSEARCH_QUERY_EXACT_MATCH_PREFIX_FACTOR:1.1} # boost multiplier when exact prefix + caseSensitivityFactor: ${ELASTICSEARCH_QUERY_EXACT_MATCH_CASE_FACTOR:0.0} # stacked boost multiplier when case mismatch enableStructured: ${ELASTICSEARCH_QUERY_EXACT_MATCH_ENABLE_STRUCTURED:true} # enable exact match on structured search wordGram: twoGramFactor: ${ELASTICSEARCH_QUERY_TWO_GRAM_FACTOR:1.2} # boost multiplier when match on 2-gram tokens @@ -230,8 +230,8 @@ elasticsearch: urnFactor: ${ELASTICSEARCH_QUERY_PARTIAL_URN_FACTOR:0.5} # multiplier on Urn token match, a partial match on Urn > non-Urn is assumed factor: ${ELASTICSEARCH_QUERY_PARTIAL_FACTOR:0.4} # multiplier on possible non-Urn token match custom: - enabled: ${ELASTICSEARCH_QUERY_CUSTOM_CONFIG_ENABLED:false} - file: ${ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE:search_config.yml} + enabled: ${ELASTICSEARCH_QUERY_CUSTOM_CONFIG_ENABLED:true} + file: ${ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE:search_config.yaml} graph: timeoutSeconds: ${ELASTICSEARCH_SEARCH_GRAPH_TIMEOUT_SECONDS:50} # graph dao timeout seconds batchSize: ${ELASTICSEARCH_SEARCH_GRAPH_BATCH_SIZE:1000} # graph dao batch size @@ -370,6 +370,7 @@ featureFlags: nestedDomainsEnabled: ${NESTED_DOMAINS_ENABLED:true} # Enables the nested Domains feature that allows users to have sub-Domains. If this is off, Domains appear "flat" again schemaFieldEntityFetchEnabled: ${SCHEMA_FIELD_ENTITY_FETCH_ENABLED:true} # Enables fetching for schema field entities from the database when we hydrate them on schema fields businessAttributeEntityEnabled: ${BUSINESS_ATTRIBUTE_ENTITY_ENABLED:false} # Enables business attribute entity which can be associated with field of dataset + dataContractsEnabled: ${DATA_CONTRACTS_ENABLED:true} # Enables the Data Contracts feature (Tab) in the UI entityChangeEvents: enabled: ${ENABLE_ENTITY_CHANGE_EVENTS_HOOK:true} @@ -386,6 +387,7 @@ entityClient: restli: get: batchSize: ${ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE:100} # limited to prevent exceeding restli URI size limit + batchConcurrency: ${ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY:2} # parallel threads usageClient: retryInterval: ${USAGE_CLIENT_RETRY_INTERVAL:2} @@ -460,3 +462,28 @@ forms: businessAttribute: fetchRelatedEntitiesCount: ${BUSINESS_ATTRIBUTE_RELATED_ENTITIES_COUNT:20000} fetchRelatedEntitiesBatchSize: ${BUSINESS_ATTRIBUTE_RELATED_ENTITIES_BATCH_SIZE:1000} + threadCount: ${BUSINESS_ATTRIBUTE_PROPAGATION_CONCURRENCY_THREAD_COUNT:-1} # Thread Pool size, default 2 * # of cores + keepAliveTime: ${BUSINESS_ATTRIBUTE_PROPAGATION_CONCURRENCY_KEEP_ALIVE:60} # Number of seconds to keep inactive threads alive + +metadataChangeProposal: + throttle: + updateIntervalMs: ${MCP_THROTTLE_UPDATE_INTERVAL_MS:60000} + + # Versioned MCL topic + versioned: + # Whether to throttle MCP processing based on MCL backlog + enabled: ${MCP_VERSIONED_THROTTLE_ENABLED:false} + threshold: ${MCP_VERSIONED_THRESHOLD:4000} # throttle threshold + maxAttempts: ${MCP_VERSIONED_MAX_ATTEMPTS:1000} + initialIntervalMs: ${MCP_VERSIONED_INITIAL_INTERVAL_MS:100} + multiplier: ${MCP_VERSIONED_MULTIPLIER:10} + maxIntervalMs: ${MCP_VERSIONED_MAX_INTERVAL_MS:30000} + # Timeseries MCL topic + timeseries: + # Whether to throttle MCP processing based on MCL backlog + enabled: ${MCP_TIMESERIES_THROTTLE_ENABLED:false} + threshold: ${MCP_TIMESERIES_THRESHOLD:4000} # throttle threshold + maxAttempts: ${MCP_TIMESERIES_MAX_ATTEMPTS:1000} + initialIntervalMs: ${MCP_TIMESERIES_INITIAL_INTERVAL_MS:100} + multiplier: ${MCP_TIMESERIES_MULTIPLIER:10} + maxIntervalMs: ${MCP_TIMESERIES_MAX_INTERVAL_MS:30000} diff --git a/metadata-service/configuration/src/main/resources/search_config.yaml b/metadata-service/configuration/src/main/resources/search_config.yaml new file mode 100644 index 0000000000000..2ffe962d39357 --- /dev/null +++ b/metadata-service/configuration/src/main/resources/search_config.yaml @@ -0,0 +1,97 @@ +# Notes: +# +# First match wins +# +# queryRegex = Java regex syntax +# +# functionScores - See the following for function score syntax +# https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-function-score-query.html + +queryConfigurations: + # Select */explore all + # Attempt to rank active incidents at the top followed by enrichment factors + - queryRegex: '[*]|' + simpleQuery: false + prefixMatchQuery: false + exactMatchQuery: false + functionScore: + functions: + - filter: + term: + hasActiveIncidents: + value: true + weight: 2.0 + - filter: + term: + hasDescription: + value: true + weight: 1.25 + - filter: + term: + hasOwners: + value: true + weight: 1.25 + - filter: + term: + hasDomain: + value: true + weight: 1.1 + - filter: + term: + hasGlossaryTerms: + value: true + weight: 1.1 + - filter: + term: + hasTags: + value: true + weight: 1.1 + - filter: + term: + hasRowCount: + value: true + weight: 1.05 + - filter: + term: + hasColumnCount: + value: true + weight: 1.05 + - filter: + term: + deprecated: + value: true + weight: 0.25 + score_mode: multiply + boost_mode: replace + + # Criteria for exact-match only + # Contains quotes, is a single term with `_`, `.`, or `-` (normally consider for tokenization) then use exact match query + - queryRegex: >- + ^["'].+["']$|^[a-zA-Z0-9]\S+[_.-]\S+[a-zA-Z0-9]$ + simpleQuery: false + prefixMatchQuery: false + exactMatchQuery: true + functionScore: + functions: + - filter: + term: + deprecated: + value: true + weight: 0.25 + score_mode: multiply + boost_mode: multiply + + # default + - queryRegex: .* + simpleQuery: true + prefixMatchQuery: true + exactMatchQuery: true + functionScore: + functions: + - filter: + term: + deprecated: + value: true + weight: 0.25 + score_mode: multiply + boost_mode: multiply diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/assertions/AssertionServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/assertions/AssertionServiceFactory.java new file mode 100644 index 0000000000000..96fb2219bf22c --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/assertions/AssertionServiceFactory.java @@ -0,0 +1,28 @@ +package com.linkedin.gms.factory.assertions; + +import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.metadata.graph.GraphClient; +import com.linkedin.metadata.service.AssertionService; +import javax.annotation.Nonnull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Scope; + +@Configuration +public class AssertionServiceFactory { + + @Autowired + @Qualifier("graphClient") + private GraphClient _graphClient; + + @Bean(name = "assertionService") + @Scope("singleton") + @Nonnull + protected AssertionService getInstance( + @Qualifier("systemEntityClient") final SystemEntityClient systemEntityClient) + throws Exception { + return new AssertionService(systemEntityClient, _graphClient); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java index 9381e24fabab6..08adbd54730a7 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java @@ -7,6 +7,7 @@ import com.linkedin.metadata.config.EbeanConfiguration; import com.linkedin.metadata.config.GraphQLConfiguration; import com.linkedin.metadata.config.IngestionConfiguration; +import com.linkedin.metadata.config.MetadataChangeProposalConfig; import com.linkedin.metadata.config.SystemUpdateConfiguration; import com.linkedin.metadata.config.TestsConfiguration; import com.linkedin.metadata.config.ViewsConfiguration; @@ -80,4 +81,7 @@ public class ConfigurationProvider { /** GraphQL Configurations */ private GraphQLConfiguration graphQL; + + /** MCP throttling configuration */ + private MetadataChangeProposalConfig metadataChangeProposal; } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/context/SystemOperationContextFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/context/SystemOperationContextFactory.java index bb59949aa44bb..6870d10c87001 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/context/SystemOperationContextFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/context/SystemOperationContextFactory.java @@ -9,6 +9,8 @@ import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.EntityServiceAspectRetriever; import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.search.SearchServiceSearchRetriever; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.metadata.context.OperationContextConfig; import io.datahubproject.metadata.context.RetrieverContext; @@ -42,7 +44,8 @@ protected OperationContext javaSystemOperationContext( @Nonnull final EntityRegistry entityRegistry, @Nonnull final EntityService entityService, @Nonnull final RestrictedService restrictedService, - @Nonnull final GraphRetriever graphRetriever) { + @Nonnull final GraphRetriever graphRetriever, + @Nonnull final SearchService searchService) { EntityServiceAspectRetriever entityServiceAspectRetriever = EntityServiceAspectRetriever.builder() @@ -50,6 +53,9 @@ protected OperationContext javaSystemOperationContext( .entityService(entityService) .build(); + SearchServiceSearchRetriever searchServiceSearchRetriever = + SearchServiceSearchRetriever.builder().searchService(searchService).build(); + OperationContext systemOperationContext = OperationContext.asSystem( operationContextConfig, @@ -60,15 +66,17 @@ protected OperationContext javaSystemOperationContext( RetrieverContext.builder() .aspectRetriever(entityServiceAspectRetriever) .graphRetriever(graphRetriever) + .searchRetriever(searchServiceSearchRetriever) .build()); entityServiceAspectRetriever.setSystemOperationContext(systemOperationContext); + searchServiceSearchRetriever.setSystemOperationContext(systemOperationContext); return systemOperationContext; } /** - * Used outside of GMS + * Used outside GMS * *

Entity Client and Aspect Retriever implemented by Restli call to GMS Entity Client and * Aspect Retriever client-side caching enabled @@ -82,11 +90,15 @@ protected OperationContext restliSystemOperationContext( @Nonnull @Qualifier("systemAuthentication") final Authentication systemAuthentication, @Nonnull final OperationContextConfig operationContextConfig, @Nonnull final RestrictedService restrictedService, - @Nonnull final GraphRetriever graphRetriever) { + @Nonnull final GraphRetriever graphRetriever, + @Nonnull final SearchService searchService) { EntityClientAspectRetriever entityServiceAspectRetriever = EntityClientAspectRetriever.builder().entityClient(systemEntityClient).build(); + SearchServiceSearchRetriever searchServiceSearchRetriever = + SearchServiceSearchRetriever.builder().searchService(searchService).build(); + OperationContext systemOperationContext = OperationContext.asSystem( operationContextConfig, @@ -97,9 +109,11 @@ protected OperationContext restliSystemOperationContext( RetrieverContext.builder() .aspectRetriever(entityServiceAspectRetriever) .graphRetriever(graphRetriever) + .searchRetriever(searchServiceSearchRetriever) .build()); entityServiceAspectRetriever.setSystemOperationContext(systemOperationContext); + searchServiceSearchRetriever.setSystemOperationContext(systemOperationContext); return systemOperationContext; } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/DeleteEntityServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/DeleteEntityServiceFactory.java index 6bc2d3c7be63f..11eb75009dc09 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/DeleteEntityServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/DeleteEntityServiceFactory.java @@ -3,6 +3,7 @@ import com.linkedin.metadata.entity.DeleteEntityService; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphService; +import com.linkedin.metadata.search.EntitySearchService; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -21,9 +22,13 @@ public class DeleteEntityServiceFactory { @Qualifier("graphService") private GraphService _graphService; + @Autowired + @Qualifier("entitySearchService") + private EntitySearchService _entitySearchService; + @Bean(name = "deleteEntityService") @Nonnull protected DeleteEntityService createDeleteEntityService() { - return new DeleteEntityService(_entityService, _graphService); + return new DeleteEntityService(_entityService, _graphService, _entitySearchService); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/RestliEntityClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/RestliEntityClientFactory.java index 2d9f570e1b07d..9e7255bf43a34 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/RestliEntityClientFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/RestliEntityClientFactory.java @@ -30,7 +30,8 @@ public EntityClient entityClient( @Value("${datahub.gms.sslContext.protocol}") String gmsSslProtocol, @Value("${entityClient.retryInterval:2}") int retryInterval, @Value("${entityClient.numRetries:3}") int numRetries, - final @Value("${entityClient.restli.get.batchSize:150}") int batchGetV2Size) { + final @Value("${entityClient.restli.get.batchSize}") int batchGetV2Size, + final @Value("${entityClient.restli.get.batchConcurrency}") int batchGetV2Concurrency) { final Client restClient; if (gmsUri != null) { restClient = DefaultRestliClientFactory.getRestLiClient(URI.create(gmsUri), gmsSslProtocol); @@ -39,7 +40,11 @@ public EntityClient entityClient( DefaultRestliClientFactory.getRestLiClient(gmsHost, gmsPort, gmsUseSSL, gmsSslProtocol); } return new RestliEntityClient( - restClient, new ExponentialBackoff(retryInterval), numRetries, batchGetV2Size); + restClient, + new ExponentialBackoff(retryInterval), + numRetries, + batchGetV2Size, + batchGetV2Concurrency); } @Bean("systemEntityClient") @@ -53,7 +58,8 @@ public SystemEntityClient systemEntityClient( @Value("${entityClient.retryInterval:2}") int retryInterval, @Value("${entityClient.numRetries:3}") int numRetries, final EntityClientCacheConfig entityClientCacheConfig, - final @Value("${entityClient.restli.get.batchSize:150}") int batchGetV2Size) { + final @Value("${entityClient.restli.get.batchSize}") int batchGetV2Size, + final @Value("${entityClient.restli.get.batchConcurrency}") int batchGetV2Concurrency) { final Client restClient; if (gmsUri != null) { @@ -67,6 +73,7 @@ public SystemEntityClient systemEntityClient( new ExponentialBackoff(retryInterval), numRetries, entityClientCacheConfig, - batchGetV2Size); + batchGetV2Size, + batchGetV2Concurrency); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java index 8c0a079f1e61d..3229f12f9021d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java @@ -14,6 +14,7 @@ import com.linkedin.datahub.graphql.concurrency.GraphQLWorkerPoolThreadFactory; import com.linkedin.entity.client.EntityClient; import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.gms.factory.assertions.AssertionServiceFactory; import com.linkedin.gms.factory.auth.DataHubTokenServiceFactory; import com.linkedin.gms.factory.common.GitVersionFactory; import com.linkedin.gms.factory.common.IndexConventionFactory; @@ -31,6 +32,7 @@ import com.linkedin.metadata.graph.SiblingGraphService; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.recommendation.RecommendationsService; +import com.linkedin.metadata.service.AssertionService; import com.linkedin.metadata.service.BusinessAttributeService; import com.linkedin.metadata.service.DataProductService; import com.linkedin.metadata.service.ERModelRelationshipService; @@ -68,7 +70,8 @@ EntityRegistryFactory.class, DataHubTokenServiceFactory.class, GitVersionFactory.class, - SiblingGraphServiceFactory.class + SiblingGraphServiceFactory.class, + AssertionServiceFactory.class, }) public class GraphQLEngineFactory { @Autowired @@ -194,6 +197,10 @@ public class GraphQLEngineFactory { @Qualifier("connectionService") private ConnectionService _connectionService; + @Autowired + @Qualifier("assertionService") + private AssertionService assertionService; + @Bean(name = "graphQLEngine") @Nonnull protected GraphQLEngine graphQLEngine( @@ -249,6 +256,7 @@ protected GraphQLEngine graphQLEngine( args.setGraphQLQueryDepthLimit(configProvider.getGraphQL().getQuery().getDepthLimit()); args.setBusinessAttributeService(businessAttributeService); args.setConnectionService(_connectionService); + args.setAssertionService(assertionService); return new GmsGraphQLEngine(args).builder().build(); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java index 5844dc4a8f72a..6a2b9f511b79f 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java @@ -44,7 +44,7 @@ public static Map buildProducerProperties( Arrays.asList(kafkaConfiguration.getBootstrapServers().split(","))); } // else we rely on KafkaProperties which defaults to localhost:9092 - Map props = properties.buildProducerProperties(); + Map props = properties.buildProducerProperties(null); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, schemaRegistryConfig.getSerializer()); diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java index d5210213185be..9501b03482d04 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java @@ -73,7 +73,7 @@ private static Map buildCustomizedProperties( Arrays.asList(kafkaConfiguration.getBootstrapServers().split(","))); } // else we rely on KafkaProperties which defaults to localhost:9092 - Map customizedProperties = baseKafkaProperties.buildConsumerProperties(); + Map customizedProperties = baseKafkaProperties.buildConsumerProperties(null); customizedProperties.put( ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ErrorHandlingDeserializer.class); customizedProperties.put( diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java index 3a6c9770fd362..0193ded97f81b 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java @@ -44,7 +44,7 @@ protected KafkaListenerContainerFactory createInstance( Arrays.asList(kafkaConfiguration.getBootstrapServers().split(","))); } // else we rely on KafkaProperties which defaults to localhost:9092 - Map customizedProperties = properties.buildConsumerProperties(); + Map customizedProperties = properties.buildConsumerProperties(null); customizedProperties.put( ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, kafkaConfiguration.getConsumer().getMaxPartitionFetchBytes()); diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/throttle/KafkaProducerThrottleFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/throttle/KafkaProducerThrottleFactory.java new file mode 100644 index 0000000000000..1eaff82fd517f --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/throttle/KafkaProducerThrottleFactory.java @@ -0,0 +1,93 @@ +package com.linkedin.gms.factory.kafka.throttle; + +import com.datahub.metadata.dao.producer.KafkaProducerThrottle; +import com.linkedin.gms.factory.config.ConfigurationProvider; +import com.linkedin.metadata.config.MetadataChangeProposalConfig; +import com.linkedin.metadata.config.kafka.KafkaConfiguration; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.mxe.Topics; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.KafkaAdminClient; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.config.KafkaListenerEndpointRegistry; +import org.springframework.kafka.listener.MessageListenerContainer; + +@Slf4j +@Configuration +public class KafkaProducerThrottleFactory { + + @Value("${METADATA_CHANGE_LOG_KAFKA_CONSUMER_GROUP_ID:generic-mae-consumer-job-client}") + private String maeConsumerGroupId; + + @Value("${METADATA_CHANGE_PROPOSAL_KAFKA_CONSUMER_GROUP_ID:generic-mce-consumer-job-client}") + private String mceConsumerGroupId; + + @Value("${METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME:" + Topics.METADATA_CHANGE_LOG_VERSIONED + "}") + private String versionedTopicName; + + @Value( + "${METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME:" + Topics.METADATA_CHANGE_LOG_TIMESERIES + "}") + private String timeseriesTopicName; + + @Bean + public KafkaProducerThrottle kafkaProducerThrottle( + @Qualifier("configurationProvider") ConfigurationProvider provider, + final KafkaProperties kafkaProperties, + final EntityRegistry entityRegistry, + final KafkaListenerEndpointRegistry registry) { + + KafkaConfiguration kafkaConfiguration = provider.getKafka(); + MetadataChangeProposalConfig mcpConfig = provider.getMetadataChangeProposal(); + + return KafkaProducerThrottle.builder() + .entityRegistry(entityRegistry) + .kafkaAdmin(kafkaAdmin(kafkaConfiguration, kafkaProperties)) + .config(mcpConfig.getThrottle()) + .mclConsumerGroupId(maeConsumerGroupId) + .timeseriesTopicName(timeseriesTopicName) + .versionedTopicName(versionedTopicName) + .pauseConsumer( + (pause) -> { + Optional container = + Optional.ofNullable(registry.getListenerContainer(mceConsumerGroupId)); + if (container.isEmpty()) { + log.warn( + "Expected container was missing: {} throttling is not possible.", + mceConsumerGroupId); + } else { + if (pause) { + container.ifPresent(MessageListenerContainer::pause); + } else { + container.ifPresent(MessageListenerContainer::resume); + } + } + }) + .build() + .start(); + } + + private static AdminClient kafkaAdmin( + KafkaConfiguration kafkaConfiguration, final KafkaProperties kafkaProperties) { + Map adminProperties = new HashMap<>(kafkaProperties.buildAdminProperties(null)); + + // KAFKA_BOOTSTRAP_SERVER has precedence over SPRING_KAFKA_BOOTSTRAP_SERVERS + if (kafkaConfiguration.getBootstrapServers() != null + && !kafkaConfiguration.getBootstrapServers().isEmpty()) { + adminProperties.put( + AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, + Arrays.asList(kafkaConfiguration.getBootstrapServers().split(","))); + } // else we rely on KafkaProperties which defaults to localhost:9092 or environment variables + + return KafkaAdminClient.create(adminProperties); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java index ef7e8a8c6a8ac..4513d9f065dcb 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java @@ -57,6 +57,9 @@ public class ElasticSearchIndexBuilderFactory { @Value("#{new Boolean('${elasticsearch.index.enableMappingsReindex}')}") private boolean enableMappingsReindex; + @Value("#{new Boolean('${structuredProperties.systemUpdateEnabled}')}") + private boolean enableStructuredPropertiesReindex; + @Bean(name = "elasticSearchIndexSettingsOverrides") @Nonnull protected Map> getIndexSettingsOverrides( @@ -85,6 +88,7 @@ protected ESIndexBuilder getInstance( overrides, enableSettingsReindex, enableMappingsReindex, + enableStructuredPropertiesReindex, configurationProvider.getElasticSearch(), gitVersion); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java index f242106e3fe4b..638d1a6fd0c8a 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java @@ -84,7 +84,7 @@ public void execute(@Nonnull OperationContext systemOperationContext) throws Exc .aspectName(DATA_PLATFORM_INSTANCE_ASPECT_NAME) .recordTemplate(dataPlatformInstance.get()) .auditStamp(aspectAuditStamp) - .build(systemOperationContext.getRetrieverContext().get().getAspectRetriever())); + .build(systemOperationContext.getAspectRetrieverOpt().get())); } } diff --git a/metadata-service/factories/src/main/resources/search_config.yml b/metadata-service/factories/src/main/resources/search_config.yml deleted file mode 100644 index 331e7d8571476..0000000000000 --- a/metadata-service/factories/src/main/resources/search_config.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Notes: -# -# First match wins -# -# queryRegex = Java regex syntax -# -# functionScores - See the following for function score syntax -# https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-function-score-query.html - -queryConfigurations: - # Select * - - queryRegex: '[*]|' - simpleQuery: false - prefixMatchQuery: false - exactMatchQuery: false - boolQuery: - must_not: - term: - deprecated: - value: true - functionScore: - functions: - - filter: - term: - materialized: - value: true - weight: 0.8 - score_mode: multiply - boost_mode: multiply - - # Criteria for exact-match only - # Contains quoted then use exact match query - - queryRegex: >- - ["'].+["'] - simpleQuery: false - prefixMatchQuery: true - exactMatchQuery: true - functionScore: - functions: - - filter: - term: - materialized: - value: true - weight: 0.8 - - filter: - term: - deprecated: - value: true - weight: 0 - score_mode: multiply - boost_mode: multiply - - # default - - queryRegex: .* - simpleQuery: true - prefixMatchQuery: true - exactMatchQuery: true - boolQuery: - must_not: - term: - deprecated: - value: true - functionScore: - functions: - - filter: - term: - materialized: - value: true - weight: 0.8 - score_mode: multiply - boost_mode: multiply diff --git a/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/GraphQLController.java b/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/GraphQLController.java index af69dce89041e..2f383b1956313 100644 --- a/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/GraphQLController.java +++ b/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/GraphQLController.java @@ -59,7 +59,8 @@ public GraphQLController() { private static final int MAX_LOG_WIDTH = 512; @PostMapping(value = "/graphql", produces = "application/json;charset=utf-8") - CompletableFuture> postGraphQL(HttpEntity httpEntity) { + CompletableFuture> postGraphQL( + HttpServletRequest request, HttpEntity httpEntity) { String jsonStr = httpEntity.getBody(); ObjectMapper mapper = new ObjectMapper(); @@ -117,13 +118,18 @@ CompletableFuture> postGraphQL(HttpEntity httpEnt SpringQueryContext context = new SpringQueryContext( - true, authentication, _authorizerChain, systemOperationContext, query, variables); + true, + authentication, + _authorizerChain, + systemOperationContext, + request, + operationName, + query, + variables); Span.current().setAttribute("actor.urn", context.getActorUrn()); - // operationName is an optional field only required if multiple operations are present - final String queryName = operationName != null ? operationName : context.getQueryName(); final String threadName = Thread.currentThread().getName(); - log.info("Processing request, operation: {}, actor urn: {}", queryName, context.getActorUrn()); + final String queryName = context.getQueryName(); log.debug("Query: {}, variables: {}", query, variables); return GraphQLConcurrencyUtils.supplyAsync( diff --git a/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/SpringQueryContext.java b/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/SpringQueryContext.java index 591e1158879d4..8aa22efb785e2 100644 --- a/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/SpringQueryContext.java +++ b/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/SpringQueryContext.java @@ -7,8 +7,10 @@ import graphql.parser.Parser; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.metadata.context.RequestContext; +import jakarta.servlet.http.HttpServletRequest; import java.util.Map; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import lombok.Getter; @Getter @@ -25,26 +27,33 @@ public SpringQueryContext( final Authentication authentication, final Authorizer authorizer, @Nonnull final OperationContext systemOperationContext, + @Nonnull final HttpServletRequest request, + @Nullable final String operationName, String jsonQuery, Map variables) { this.isAuthenticated = isAuthenticated; this.authentication = authentication; this.authorizer = authorizer; + // operationName is an optional field only required if multiple operations are present this.queryName = - new Parser() - .parseDocument(jsonQuery).getDefinitions().stream() - .filter(def -> def instanceof OperationDefinition) - .map(def -> (OperationDefinition) def) - .filter(opDef -> opDef.getOperation().equals(OperationDefinition.Operation.QUERY)) - .findFirst() - .map(OperationDefinition::getName) - .orElse("graphql"); + operationName != null + ? operationName + : new Parser() + .parseDocument(jsonQuery).getDefinitions().stream() + .filter(def -> def instanceof OperationDefinition) + .map(def -> (OperationDefinition) def) + .filter( + opDef -> opDef.getOperation().equals(OperationDefinition.Operation.QUERY)) + .findFirst() + .map(OperationDefinition::getName) + .orElse("graphql"); this.operationContext = OperationContext.asSession( systemOperationContext, - RequestContext.builder().buildGraphql(queryName, variables), + RequestContext.builder() + .buildGraphql(authentication.getActor().toUrnStr(), request, queryName, variables), authorizer, authentication, true); diff --git a/metadata-service/openapi-analytics-servlet/build.gradle b/metadata-service/openapi-analytics-servlet/build.gradle index 7c6568fa78f64..5d1372a293775 100644 --- a/metadata-service/openapi-analytics-servlet/build.gradle +++ b/metadata-service/openapi-analytics-servlet/build.gradle @@ -9,6 +9,7 @@ dependencies { implementation project(':metadata-service:auth-impl') implementation project(':metadata-service:factories') implementation project(':metadata-service:openapi-servlet') + implementation project(':metadata-service:openapi-servlet:models') implementation project(':metadata-models') implementation externalDependency.springBoot @@ -18,7 +19,7 @@ dependencies { implementation externalDependency.springWebMVC implementation externalDependency.springBeans implementation externalDependency.springContext - + implementation externalDependency.servletApi implementation externalDependency.reflections implementation externalDependency.slf4jApi compileOnly externalDependency.lombok diff --git a/metadata-service/openapi-analytics-servlet/src/main/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImpl.java b/metadata-service/openapi-analytics-servlet/src/main/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImpl.java index 53df1e50ffbca..dc6d4a33f936e 100644 --- a/metadata-service/openapi-analytics-servlet/src/main/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImpl.java +++ b/metadata-service/openapi-analytics-servlet/src/main/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImpl.java @@ -12,6 +12,7 @@ import io.datahubproject.metadata.context.RequestContext; import io.datahubproject.openapi.exception.UnauthorizedException; import io.datahubproject.openapi.v2.generated.controller.DatahubUsageEventsApiDelegate; +import jakarta.servlet.http.HttpServletRequest; import java.util.List; import java.util.Objects; import org.springframework.beans.factory.annotation.Autowired; @@ -27,6 +28,8 @@ public class DatahubUsageEventsImpl implements DatahubUsageEventsApiDelegate { @Qualifier("systemOperationContext") OperationContext systemOperationContext; + @Autowired private HttpServletRequest request; + public static final String DATAHUB_USAGE_INDEX = "datahub_usage_event"; @Override @@ -36,7 +39,8 @@ public ResponseEntity raw(String body) { OperationContext opContext = OperationContext.asSession( systemOperationContext, - RequestContext.builder().buildOpenapi("raw", List.of()), + RequestContext.builder() + .buildOpenapi(authentication.getActor().toUrnStr(), request, "raw", List.of()), _authorizationChain, authentication, true); diff --git a/metadata-service/openapi-entity-servlet/build.gradle b/metadata-service/openapi-entity-servlet/build.gradle index 016ac6693f55b..4c2d587a81fd7 100644 --- a/metadata-service/openapi-entity-servlet/build.gradle +++ b/metadata-service/openapi-entity-servlet/build.gradle @@ -9,6 +9,7 @@ dependencies { implementation project(':metadata-service:auth-impl') implementation project(':metadata-service:factories') implementation project(':metadata-service:openapi-servlet') + implementation project(':metadata-service:openapi-servlet:models') implementation project(':metadata-models') implementation externalDependency.servletApi diff --git a/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImpl.java b/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImpl.java index bd4d68834b9e8..9e671bfb7e01e 100644 --- a/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImpl.java +++ b/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImpl.java @@ -59,6 +59,7 @@ import io.datahubproject.openapi.generated.StatusAspectResponseV2; import io.datahubproject.openapi.util.OpenApiEntitiesUtil; import io.datahubproject.openapi.v1.entities.EntitiesController; +import jakarta.servlet.http.HttpServletRequest; import java.net.URISyntaxException; import java.util.List; import java.util.Map; @@ -69,12 +70,16 @@ import javax.annotation.Nullable; import javax.validation.Valid; import javax.validation.constraints.Min; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpEntity; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @Slf4j +@Accessors(chain = true) public class EntityApiDelegateImpl { private final OperationContext systemOperationContext; private final EntityRegistry _entityRegistry; @@ -85,6 +90,7 @@ public class EntityApiDelegateImpl { private final Class _reqClazz; private final Class _respClazz; private final Class _scrollRespClazz; + @Setter @Getter private HttpServletRequest request; private static final String BUSINESS_ATTRIBUTE_ERROR_MESSAGE = "business attribute is disabled, enable it using featureflag : BUSINESS_ATTRIBUTE_ENTITY_ENABLED"; @@ -92,6 +98,7 @@ public class EntityApiDelegateImpl { public EntityApiDelegateImpl( OperationContext systemOperationContext, + HttpServletRequest request, EntityService entityService, SearchService searchService, EntitiesController entitiesController, @@ -100,6 +107,7 @@ public EntityApiDelegateImpl( Class respClazz, Class scrollRespClazz) { this.systemOperationContext = systemOperationContext; + this.request = request; this._entityService = entityService; this._searchService = searchService; this._entityRegistry = systemOperationContext.getEntityRegistry(); @@ -119,7 +127,7 @@ public ResponseEntity get(String urn, Boolean systemMetadata, List as .map(asp -> asp.stream().distinct().toArray(String[]::new)) .orElse(null); ResponseEntity result = - _v1Controller.getEntities(new String[] {urn}, requestedAspects); + _v1Controller.getEntities(request, new String[] {urn}, requestedAspects); return ResponseEntity.of( OpenApiEntitiesUtil.convertEntity( Optional.ofNullable(result).map(HttpEntity::getBody).orElse(null), @@ -146,7 +154,7 @@ public ResponseEntity> create( throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); } } - _v1Controller.postEntities(aspects, false, createIfNotExists, createEntityIfNotExists); + _v1Controller.postEntities(request, aspects, false, createIfNotExists, createEntityIfNotExists); List responses = body.stream() .map(req -> OpenApiEntitiesUtil.convertToResponse(req, _respClazz, _entityRegistry)) @@ -158,7 +166,7 @@ public ResponseEntity delete(String urn) { if (checkBusinessAttributeFlagFromUrn(urn)) { throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); } - _v1Controller.deleteEntities(new String[] {urn}, false, false); + _v1Controller.deleteEntities(request, new String[] {urn}, false, false); return new ResponseEntity<>(HttpStatus.OK); } @@ -177,7 +185,9 @@ public ResponseEntity head(String urn) { OperationContext opContext = OperationContext.asSession( systemOperationContext, - RequestContext.builder().buildOpenapi("head", entityUrn.getEntityType()), + RequestContext.builder() + .buildOpenapi( + auth.getActor().toUrnStr(), request, "head", entityUrn.getEntityType()), _authorizationChain, auth, true); @@ -200,7 +210,7 @@ public ResponseEntity getAspect( Class aspectRespClazz) { String[] requestedAspects = new String[] {aspect}; ResponseEntity result = - _v1Controller.getEntities(new String[] {urn}, requestedAspects); + _v1Controller.getEntities(request, new String[] {urn}, requestedAspects); return ResponseEntity.of( OpenApiEntitiesUtil.convertAspect( result.getBody(), aspect, entityRespClass, aspectRespClazz, systemMetadata)); @@ -217,6 +227,7 @@ public ResponseEntity createAspect( UpsertAspectRequest aspectUpsert = OpenApiEntitiesUtil.convertAspectToUpsert(urn, body, reqClazz); _v1Controller.postEntities( + request, Stream.of(aspectUpsert).filter(Objects::nonNull).collect(Collectors.toList()), false, createIfNotExists, @@ -238,7 +249,9 @@ public ResponseEntity headAspect(String urn, String aspect) { OperationContext opContext = OperationContext.asSession( systemOperationContext, - RequestContext.builder().buildOpenapi("headAspect", entityUrn.getEntityType()), + RequestContext.builder() + .buildOpenapi( + auth.getActor().toUrnStr(), request, "headAspect", entityUrn.getEntityType()), _authorizationChain, auth, true); @@ -259,12 +272,14 @@ public ResponseEntity deleteAspect(String urn, String aspect) { OperationContext opContext = OperationContext.asSession( systemOperationContext, - RequestContext.builder().buildOpenapi("deleteAspect", entityUrn.getEntityType()), + RequestContext.builder() + .buildOpenapi( + auth.getActor().toUrnStr(), request, "deleteAspect", entityUrn.getEntityType()), _authorizationChain, auth, true); _entityService.deleteAspect(opContext, urn, aspect, Map.of(), false); - _v1Controller.deleteEntities(new String[] {urn}, false, false); + _v1Controller.deleteEntities(request, new String[] {urn}, false, false); return new ResponseEntity<>(HttpStatus.OK); } @@ -606,7 +621,9 @@ public ResponseEntity scroll( OperationContext opContext = OperationContext.asSession( systemOperationContext, - RequestContext.builder().buildOpenapi("scroll", entitySpec.getName()), + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), request, "scroll", entitySpec.getName()), _authorizationChain, authentication, true); @@ -644,7 +661,7 @@ public ResponseEntity scroll( .map(asp -> asp.stream().distinct().toArray(String[]::new)) .orElse(null); List entities = - Optional.ofNullable(_v1Controller.getEntities(urns, requestedAspects).getBody()) + Optional.ofNullable(_v1Controller.getEntities(request, urns, requestedAspects).getBody()) .map(body -> body.getResponses().entrySet()) .map( entries -> OpenApiEntitiesUtil.convertEntities(entries, _respClazz, systemMetadata)) diff --git a/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/apiController.mustache b/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/apiController.mustache index 04bc7e52c593b..fbf354ff91688 100644 --- a/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/apiController.mustache +++ b/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/apiController.mustache @@ -98,7 +98,7 @@ public class {{classname}}Controller implements {{classname}} { SearchService searchService, EntitiesController v1Controller, AuthorizerChain authorizationChain) { this.objectMapper = objectMapper; this.request = request; - this.delegate = new EntityApiDelegateImpl<{{requestClass}}, {{responseClass}}, {{scrollResponseClass}}>(systemOperationContext, entityService, searchService, v1Controller, + this.delegate = new EntityApiDelegateImpl<{{requestClass}}, {{responseClass}}, {{scrollResponseClass}}>(systemOperationContext, request, entityService, searchService, v1Controller, authorizationChain, {{requestClass}}.class, {{responseClass}}.class, {{scrollResponseClass}}.class); } {{#isJava8or11}} diff --git a/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIEntityTestConfiguration.java b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIEntityTestConfiguration.java index 3fba99a4b197f..075501c1a1071 100644 --- a/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIEntityTestConfiguration.java +++ b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIEntityTestConfiguration.java @@ -3,6 +3,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -31,8 +32,9 @@ import io.datahubproject.openapi.generated.EntityResponse; import io.datahubproject.openapi.v1.entities.EntitiesController; import io.datahubproject.openapi.v1.relationships.RelationshipsController; -import io.datahubproject.openapi.v2.controller.TimelineController; +import io.datahubproject.openapi.v2.controller.TimelineControllerV2; import io.datahubproject.test.metadata.context.TestOperationContexts; +import jakarta.servlet.http.HttpServletRequest; import java.util.Arrays; import java.util.Map; import java.util.stream.Collectors; @@ -109,11 +111,11 @@ public EntityRegistry entityRegistry() throws EntityRegistryException, Interrupt @Primary public EntitiesController entitiesController() { EntitiesController entitiesController = mock(EntitiesController.class); - when(entitiesController.getEntities(any(), any())) + when(entitiesController.getEntities(nullable(HttpServletRequest.class), any(), any())) .thenAnswer( params -> { - String[] urns = params.getArgument(0); - String[] aspects = params.getArgument(1); + String[] urns = params.getArgument(1); + String[] aspects = params.getArgument(2); return ResponseEntity.ok( UrnResponseMap.builder() .responses( @@ -126,7 +128,7 @@ public EntitiesController entitiesController() { return entitiesController; } - @MockBean public TimelineController timelineController; + @MockBean public TimelineControllerV2 timelineControllerV2; @MockBean public RelationshipsController relationshipsController; diff --git a/metadata-service/openapi-servlet/build.gradle b/metadata-service/openapi-servlet/build.gradle index bfe9d954dfc7d..e26b1ceea1a3c 100644 --- a/metadata-service/openapi-servlet/build.gradle +++ b/metadata-service/openapi-servlet/build.gradle @@ -21,6 +21,7 @@ dependencies { implementation externalDependency.springWebMVC implementation externalDependency.springBeans implementation externalDependency.springContext + implementation externalDependency.servletApi implementation externalDependency.slf4jApi compileOnly externalDependency.lombok implementation externalDependency.antlr4Runtime @@ -29,6 +30,7 @@ dependencies { implementation externalDependency.guava implementation('io.acryl:json-schema-avro:0.2.3') implementation externalDependency.jsonSchemaValidator + implementation group: 'io.github.deblockt', name: 'json-diff', version: '1.1.0' annotationProcessor externalDependency.lombok diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/client/OpenApiClient.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/client/OpenApiClient.java index 267d95f1dddbf..bdea174faa243 100644 --- a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/client/OpenApiClient.java +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/client/OpenApiClient.java @@ -2,8 +2,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.datahubproject.metadata.context.OperationContext; -import io.datahubproject.openapi.v2.models.BatchGetUrnRequest; -import io.datahubproject.openapi.v2.models.BatchGetUrnResponse; +import io.datahubproject.openapi.v2.models.BatchGetUrnRequestV2; +import io.datahubproject.openapi.v2.models.BatchGetUrnResponseV2; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -40,15 +40,16 @@ public OpenApiClient( this.systemOperationContext = systemOperationContext; } - public BatchGetUrnResponse getBatchUrnsSystemAuth(String entityName, BatchGetUrnRequest request) { + public BatchGetUrnResponseV2 getBatchUrnsSystemAuth( + String entityName, BatchGetUrnRequestV2 request) { return getBatchUrns( entityName, request, systemOperationContext.getSystemAuthentication().get().getCredentials()); } - public BatchGetUrnResponse getBatchUrns( - String entityName, BatchGetUrnRequest request, String authCredentials) { + public BatchGetUrnResponseV2 getBatchUrns( + String entityName, BatchGetUrnRequestV2 request, String authCredentials) { String url = (useSsl ? "https://" : "http://") + gmsHost + ":" + gmsPort + OPENAPI_PATH + entityName; HttpPost httpPost = new HttpPost(url); @@ -65,8 +66,8 @@ public BatchGetUrnResponse getBatchUrns( } } - private static BatchGetUrnResponse mapResponse(ClassicHttpResponse response) { - BatchGetUrnResponse serializedResponse; + private static BatchGetUrnResponseV2 mapResponse(ClassicHttpResponse response) { + BatchGetUrnResponseV2 serializedResponse; try { ByteArrayOutputStream result = new ByteArrayOutputStream(); InputStream contentStream = response.getEntity().getContent(); @@ -78,7 +79,7 @@ private static BatchGetUrnResponse mapResponse(ClassicHttpResponse response) { } serializedResponse = OBJECT_MAPPER.readValue( - result.toString(StandardCharsets.UTF_8), BatchGetUrnResponse.class); + result.toString(StandardCharsets.UTF_8), BatchGetUrnResponseV2.class); } catch (IOException e) { log.error("Wasn't able to convert response into expected type.", e); throw new RuntimeException(e); diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/exception/InvalidUrnException.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/exception/InvalidUrnException.java new file mode 100644 index 0000000000000..52a7f95f78da0 --- /dev/null +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/exception/InvalidUrnException.java @@ -0,0 +1,9 @@ +package io.datahubproject.openapi.exception; + +import java.net.URISyntaxException; + +public class InvalidUrnException extends URISyntaxException { + public InvalidUrnException(String input, String reason) { + super(input, reason); + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/exception/UnauthorizedException.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/exception/UnauthorizedException.java similarity index 100% rename from metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/exception/UnauthorizedException.java rename to metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/exception/UnauthorizedException.java diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/models/GenericAspect.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/models/GenericAspect.java new file mode 100644 index 0000000000000..eb55864400494 --- /dev/null +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/models/GenericAspect.java @@ -0,0 +1,16 @@ +package io.datahubproject.openapi.models; + +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public interface GenericAspect { + @Nonnull + Map getValue(); + + @Nullable + Map getSystemMetadata(); + + @Nullable + Map getHeaders(); +} diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/models/GenericEntity.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/models/GenericEntity.java new file mode 100644 index 0000000000000..fcb626ca60fc1 --- /dev/null +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/models/GenericEntity.java @@ -0,0 +1,7 @@ +package io.datahubproject.openapi.models; + +import java.util.Map; + +public interface GenericEntity { + Map getAspects(); +} diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/models/GenericEntityScrollResult.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/models/GenericEntityScrollResult.java new file mode 100644 index 0000000000000..59ea624e05c70 --- /dev/null +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/models/GenericEntityScrollResult.java @@ -0,0 +1,3 @@ +package io.datahubproject.openapi.models; + +public interface GenericEntityScrollResult> {} diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericScrollResult.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/models/GenericScrollResult.java similarity index 79% rename from metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericScrollResult.java rename to metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/models/GenericScrollResult.java index 2befc83c00363..7864af3bb4cdd 100644 --- a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericScrollResult.java +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/models/GenericScrollResult.java @@ -1,4 +1,4 @@ -package io.datahubproject.openapi.v2.models; +package io.datahubproject.openapi.models; import java.util.List; import lombok.Builder; diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/BatchGetUrnRequest.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/BatchGetUrnRequestV2.java similarity index 86% rename from metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/BatchGetUrnRequest.java rename to metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/BatchGetUrnRequestV2.java index 02afe8d40dd52..bb3bcd29abda7 100644 --- a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/BatchGetUrnRequest.java +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/BatchGetUrnRequestV2.java @@ -13,9 +13,9 @@ @Value @EqualsAndHashCode @Builder -@JsonDeserialize(builder = BatchGetUrnRequest.BatchGetUrnRequestBuilder.class) +@JsonDeserialize(builder = BatchGetUrnRequestV2.BatchGetUrnRequestV2Builder.class) @JsonInclude(JsonInclude.Include.NON_NULL) -public class BatchGetUrnRequest implements Serializable { +public class BatchGetUrnRequestV2 implements Serializable { @JsonProperty("urns") @Schema(required = true, description = "The list of urns to get.") List urns; diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/BatchGetUrnResponse.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/BatchGetUrnResponseV2.java similarity index 60% rename from metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/BatchGetUrnResponse.java rename to metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/BatchGetUrnResponseV2.java index 628733e4fd4ae..391a1e8627642 100644 --- a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/BatchGetUrnResponse.java +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/BatchGetUrnResponseV2.java @@ -3,6 +3,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import io.datahubproject.openapi.models.GenericAspect; +import io.datahubproject.openapi.models.GenericEntity; import io.swagger.v3.oas.annotations.media.Schema; import java.io.Serializable; import java.util.List; @@ -12,9 +14,10 @@ @Value @Builder @JsonInclude(JsonInclude.Include.NON_NULL) -@JsonDeserialize(builder = BatchGetUrnResponse.BatchGetUrnResponseBuilder.class) -public class BatchGetUrnResponse implements Serializable { +@JsonDeserialize(builder = BatchGetUrnResponseV2.BatchGetUrnResponseV2Builder.class) +public class BatchGetUrnResponseV2> + implements Serializable { @JsonProperty("entities") @Schema(description = "List of entity responses") - List entities; + List entities; } diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericAspectV2.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericAspectV2.java new file mode 100644 index 0000000000000..dd50f8d8b2a5e --- /dev/null +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericAspectV2.java @@ -0,0 +1,32 @@ +package io.datahubproject.openapi.v2.models; + +import io.datahubproject.openapi.models.GenericAspect; +import java.util.LinkedHashMap; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class GenericAspectV2 extends LinkedHashMap implements GenericAspect { + + public GenericAspectV2(Map m) { + super(m); + } + + @Nonnull + @Override + public Map getValue() { + return this; + } + + @Nullable + @Override + public Map getSystemMetadata() { + return null; + } + + @Nullable + @Override + public Map getHeaders() { + return null; + } +} diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericEntityScrollResultV2.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericEntityScrollResultV2.java new file mode 100644 index 0000000000000..221c4a519168c --- /dev/null +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericEntityScrollResultV2.java @@ -0,0 +1,14 @@ +package io.datahubproject.openapi.v2.models; + +import io.datahubproject.openapi.models.GenericEntityScrollResult; +import java.util.List; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class GenericEntityScrollResultV2 + implements GenericEntityScrollResult { + private String scrollId; + private List results; +} diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericEntity.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericEntityV2.java similarity index 71% rename from metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericEntity.java rename to metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericEntityV2.java index cb049c5ba131a..83c23f3552409 100644 --- a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericEntity.java +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v2/models/GenericEntityV2.java @@ -7,6 +7,7 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.mxe.SystemMetadata; import com.linkedin.util.Pair; +import io.datahubproject.openapi.models.GenericEntity; import io.swagger.v3.oas.annotations.media.Schema; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -23,20 +24,20 @@ @JsonInclude(JsonInclude.Include.NON_NULL) @NoArgsConstructor(force = true, access = AccessLevel.PRIVATE) @AllArgsConstructor -public class GenericEntity { +public class GenericEntityV2 implements GenericEntity { @JsonProperty("urn") @Schema(description = "Urn of the entity") private String urn; @JsonProperty("aspects") @Schema(description = "Map of aspect name to aspect") - private Map aspects; + private Map aspects; - public static class GenericEntityBuilder { + public static class GenericEntityV2Builder { - public GenericEntity build( + public GenericEntityV2 build( ObjectMapper objectMapper, Map> aspects) { - Map jsonObjectMap = + Map jsonObjectMap = aspects.entrySet().stream() .map( e -> { @@ -51,11 +52,14 @@ public GenericEntity build( if (e.getValue().getSecond() != null) { return Map.entry( e.getKey(), - Map.of( - "systemMetadata", e.getValue().getSecond(), - "value", valueMap.get("value"))); + new GenericAspectV2( + Map.of( + "systemMetadata", e.getValue().getSecond(), + "value", valueMap.get("value")))); } else { - return Map.entry(e.getKey(), Map.of("value", valueMap.get("value"))); + return Map.entry( + e.getKey(), + new GenericAspectV2(Map.of("value", valueMap.get("value")))); } } catch (IOException ex) { throw new RuntimeException(ex); @@ -63,7 +67,7 @@ public GenericEntity build( }) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - return new GenericEntity(urn, jsonObjectMap); + return new GenericEntityV2(urn, jsonObjectMap); } } } diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericAspectV3.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericAspectV3.java new file mode 100644 index 0000000000000..4db2c3288d154 --- /dev/null +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericAspectV3.java @@ -0,0 +1,22 @@ +package io.datahubproject.openapi.v3.models; + +import com.fasterxml.jackson.annotation.JsonInclude; +import io.datahubproject.openapi.models.GenericAspect; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Value; + +@EqualsAndHashCode +@Value +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +@AllArgsConstructor +public class GenericAspectV3 implements GenericAspect { + @Nonnull Map value; + @Nullable Map systemMetadata; + @Nullable Map headers; +} diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityScrollResultV3.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityScrollResultV3.java new file mode 100644 index 0000000000000..b2d096e8140bd --- /dev/null +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityScrollResultV3.java @@ -0,0 +1,14 @@ +package io.datahubproject.openapi.v3.models; + +import io.datahubproject.openapi.models.GenericEntityScrollResult; +import java.util.List; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class GenericEntityScrollResultV3 + implements GenericEntityScrollResult { + private String scrollId; + private List entities; +} diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityV3.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityV3.java new file mode 100644 index 0000000000000..3af3b25028fad --- /dev/null +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityV3.java @@ -0,0 +1,83 @@ +package io.datahubproject.openapi.v3.models; + +import com.datahub.util.RecordUtils; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.mxe.SystemMetadata; +import com.linkedin.util.Pair; +import io.datahubproject.openapi.models.GenericEntity; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode(callSuper = true) +@Data +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +@AllArgsConstructor +public class GenericEntityV3 extends LinkedHashMap + implements GenericEntity { + + public GenericEntityV3(Map m) { + super(m); + } + + @Override + public Map getAspects() { + return this.entrySet().stream() + .filter(entry -> !"urn".equals(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, entry -> (GenericAspectV3) entry.getValue())); + } + + public static class GenericEntityV3Builder { + + public GenericEntityV3 build( + ObjectMapper objectMapper, + @Nonnull Urn urn, + Map> aspects) { + Map jsonObjectMap = + aspects.entrySet().stream() + .map( + entry -> { + try { + String aspectName = entry.getKey(); + Map aspectValue = + objectMapper.readValue( + RecordUtils.toJsonString(entry.getValue().getFirst()) + .getBytes(StandardCharsets.UTF_8), + new TypeReference<>() {}); + Map systemMetadata = + entry.getValue().getSecond() != null + ? objectMapper.convertValue( + entry.getValue().getSecond(), new TypeReference<>() {}) + : null; + + return Map.entry( + aspectName, + GenericAspectV3.builder() + .value(aspectValue) + .systemMetadata(systemMetadata) + .build()); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + GenericEntityV3 genericEntityV3 = new GenericEntityV3(); + genericEntityV3.put("urn", urn.toString()); + genericEntityV3.putAll(jsonObjectMap); + return genericEntityV3; + } + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/GlobalControllerExceptionHandler.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/GlobalControllerExceptionHandler.java index f4689a9862825..0e9fcbe15b525 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/GlobalControllerExceptionHandler.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/GlobalControllerExceptionHandler.java @@ -1,5 +1,7 @@ package io.datahubproject.openapi; +import io.datahubproject.openapi.exception.InvalidUrnException; +import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.ConversionNotSupportedException; import org.springframework.core.Ordered; @@ -23,4 +25,9 @@ public GlobalControllerExceptionHandler() { public ResponseEntity handleConflict(RuntimeException ex) { return new ResponseEntity<>(ex.getMessage(), HttpStatus.BAD_REQUEST); } + + @ExceptionHandler(InvalidUrnException.class) + public static ResponseEntity> handleUrnException(InvalidUrnException e) { + return new ResponseEntity<>(Map.of("error", e.getMessage()), HttpStatus.BAD_REQUEST); + } } diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java new file mode 100644 index 0000000000000..c91c8ac987e5c --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java @@ -0,0 +1,786 @@ +package io.datahubproject.openapi.controller; + +import static com.linkedin.metadata.authorization.ApiOperation.CREATE; +import static com.linkedin.metadata.authorization.ApiOperation.DELETE; +import static com.linkedin.metadata.authorization.ApiOperation.EXISTS; +import static com.linkedin.metadata.authorization.ApiOperation.READ; +import static com.linkedin.metadata.authorization.ApiOperation.UPDATE; + +import com.datahub.authentication.Actor; +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; +import com.datahub.authorization.AuthUtil; +import com.datahub.authorization.AuthorizerChain; +import com.datahub.util.RecordUtils; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.ByteString; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.AspectRetriever; +import com.linkedin.metadata.aspect.batch.AspectsBatch; +import com.linkedin.metadata.aspect.batch.ChangeMCP; +import com.linkedin.metadata.aspect.patch.GenericJsonPatch; +import com.linkedin.metadata.aspect.patch.template.common.GenericPatchTemplate; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.IngestResult; +import com.linkedin.metadata.entity.UpdateAspectResult; +import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; +import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.query.filter.SortOrder; +import com.linkedin.metadata.search.ScrollResult; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.utils.AuditStampUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.metadata.utils.SearchUtil; +import com.linkedin.mxe.SystemMetadata; +import com.linkedin.util.Pair; +import io.datahubproject.metadata.context.OperationContext; +import io.datahubproject.metadata.context.RequestContext; +import io.datahubproject.openapi.exception.InvalidUrnException; +import io.datahubproject.openapi.exception.UnauthorizedException; +import io.datahubproject.openapi.models.GenericAspect; +import io.datahubproject.openapi.models.GenericEntity; +import io.datahubproject.openapi.models.GenericEntityScrollResult; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; +import java.lang.reflect.InvocationTargetException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PatchMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; + +public abstract class GenericEntitiesController< + A extends GenericAspect, + E extends GenericEntity, + S extends GenericEntityScrollResult> { + public static final String NOT_FOUND_HEADER = "Not-Found-Reason"; + protected static final SearchFlags DEFAULT_SEARCH_FLAGS = + new SearchFlags().setFulltext(false).setSkipAggregates(true).setSkipHighlighting(true); + + @Autowired protected EntityRegistry entityRegistry; + @Autowired protected SearchService searchService; + @Autowired protected EntityService entityService; + @Autowired protected AuthorizerChain authorizationChain; + @Autowired protected ObjectMapper objectMapper; + + @Qualifier("systemOperationContext") + @Autowired + protected OperationContext systemOperationContext; + + /** + * Returns scroll result entities + * + * @param searchEntities the entities to contain in the result + * @param aspectNames the aspect names present + * @param withSystemMetadata whether to include system metadata in the result + * @param scrollId the pagination token + * @return result containing entities/aspects + * @throws URISyntaxException parsing error + */ + protected abstract S buildScrollResult( + @Nonnull OperationContext opContext, + SearchEntityArray searchEntities, + Set aspectNames, + boolean withSystemMetadata, + @Nullable String scrollId) + throws URISyntaxException; + + protected List buildEntityList( + @Nonnull OperationContext opContext, + List urns, + Set aspectNames, + boolean withSystemMetadata) + throws URISyntaxException { + + LinkedHashMap> versionMap = + resolveAspectNames( + urns.stream() + .map( + urn -> + Map.entry( + urn, + aspectNames.stream() + .map(aspectName -> Map.entry(aspectName, 0L)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))) + .collect( + Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue, + (a, b) -> { + throw new IllegalStateException("Duplicate key"); + }, + LinkedHashMap::new)), + 0L); + return buildEntityVersionedAspectList(opContext, versionMap, withSystemMetadata); + } + + protected abstract List buildEntityVersionedAspectList( + @Nonnull OperationContext opContext, + LinkedHashMap> urnAspectVersions, + boolean withSystemMetadata) + throws URISyntaxException; + + protected abstract List buildEntityList( + Set ingestResults, boolean withSystemMetadata); + + protected abstract E buildGenericEntity( + @Nonnull String aspectName, + @Nonnull UpdateAspectResult updateAspectResult, + boolean withSystemMetadata); + + protected abstract AspectsBatch toMCPBatch( + @Nonnull OperationContext opContext, String entityArrayList, Actor actor) + throws JsonProcessingException, InvalidUrnException; + + @Tag(name = "Generic Entities", description = "API for interacting with generic entities.") + @GetMapping(value = "/{entityName}", produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Scroll entities") + public ResponseEntity getEntities( + HttpServletRequest request, + @PathVariable("entityName") String entityName, + @RequestParam(value = "aspectNames", defaultValue = "") Set aspects1, + @RequestParam(value = "aspects", defaultValue = "") Set aspects2, + @RequestParam(value = "count", defaultValue = "10") Integer count, + @RequestParam(value = "query", defaultValue = "*") String query, + @RequestParam(value = "scrollId", required = false) String scrollId, + @RequestParam(value = "sort", required = false, defaultValue = "urn") String sortField, + @RequestParam(value = "sortOrder", required = false, defaultValue = "ASCENDING") + String sortOrder, + @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") + Boolean withSystemMetadata, + @RequestParam(value = "skipCache", required = false, defaultValue = "false") + Boolean skipCache, + @RequestParam(value = "includeSoftDelete", required = false, defaultValue = "false") + Boolean includeSoftDelete) + throws URISyntaxException { + + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); + Authentication authentication = AuthenticationContext.getAuthentication(); + + if (!AuthUtil.isAPIAuthorizedEntityType(authentication, authorizationChain, READ, entityName)) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + " is unauthorized to " + READ + " entities."); + } + + OperationContext opContext = + OperationContext.asSession( + systemOperationContext, + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), request, "getEntities", entityName), + authorizationChain, + authentication, + true); + + // TODO: support additional and multiple sort params + SortCriterion sortCriterion = SearchUtil.sortBy(sortField, SortOrder.valueOf(sortOrder)); + + ScrollResult result = + searchService.scrollAcrossEntities( + opContext + .withSearchFlags(flags -> DEFAULT_SEARCH_FLAGS) + .withSearchFlags(flags -> flags.setSkipCache(skipCache)) + .withSearchFlags(flags -> flags.setIncludeSoftDeleted(includeSoftDelete)), + List.of(entitySpec.getName()), + query, + null, + sortCriterion, + scrollId, + null, + count); + + if (!AuthUtil.isAPIAuthorizedResult(authentication, authorizationChain, result)) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + " is unauthorized to " + READ + " entities."); + } + + return ResponseEntity.ok( + buildScrollResult( + opContext, + result.getEntities(), + ImmutableSet.builder().addAll(aspects1).addAll(aspects2).build(), + withSystemMetadata, + result.getScrollId())); + } + + @Tag(name = "Generic Entities") + @GetMapping( + value = "/{entityName}/{entityUrn:urn:li:.+}", + produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity getEntity( + HttpServletRequest request, + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn, + @RequestParam(value = "aspectNames", defaultValue = "") Set aspects1, + @RequestParam(value = "aspects", defaultValue = "") Set aspects2, + @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") + Boolean withSystemMetadata) + throws URISyntaxException { + + Urn urn = validatedUrn(entityUrn); + Authentication authentication = AuthenticationContext.getAuthentication(); + if (!AuthUtil.isAPIAuthorizedEntityUrns( + authentication, authorizationChain, READ, List.of(urn))) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + " is unauthorized to " + READ + " entities."); + } + OperationContext opContext = + OperationContext.asSession( + systemOperationContext, + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), request, "getEntity", entityName), + authorizationChain, + authentication, + true); + + return buildEntityList( + opContext, + List.of(urn), + ImmutableSet.builder().addAll(aspects1).addAll(aspects2).build(), + withSystemMetadata) + .stream() + .findFirst() + .map(ResponseEntity::ok) + .orElse(ResponseEntity.notFound().header(NOT_FOUND_HEADER, "ENTITY").build()); + } + + @Tag(name = "Generic Entities") + @RequestMapping( + value = "/{entityName}/{entityUrn:urn:li:.+}", + method = {RequestMethod.HEAD}) + @Operation(summary = "Entity exists") + public ResponseEntity headEntity( + HttpServletRequest request, + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn, + @PathVariable(value = "includeSoftDelete", required = false) Boolean includeSoftDelete) + throws InvalidUrnException { + + Urn urn = validatedUrn(entityUrn); + Authentication authentication = AuthenticationContext.getAuthentication(); + if (!AuthUtil.isAPIAuthorizedEntityUrns( + authentication, authorizationChain, EXISTS, List.of(urn))) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + " is unauthorized to " + EXISTS + " entities."); + } + OperationContext opContext = + OperationContext.asSession( + systemOperationContext, + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), request, "headEntity", entityName), + authorizationChain, + authentication, + true); + + return exists(opContext, urn, null, includeSoftDelete) + ? ResponseEntity.noContent().build() + : ResponseEntity.notFound().build(); + } + + @Tag(name = "Generic Aspects", description = "API for generic aspects.") + @GetMapping( + value = "/{entityName}/{entityUrn:urn:li:.+}/{aspectName}", + produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Get an entity's generic aspect.") + public ResponseEntity getAspect( + HttpServletRequest request, + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn, + @PathVariable("aspectName") String aspectName, + @RequestParam(value = "version", required = false, defaultValue = "0") Long version, + @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") + Boolean withSystemMetadata) + throws URISyntaxException { + + Urn urn = validatedUrn(entityUrn); + Authentication authentication = AuthenticationContext.getAuthentication(); + if (!AuthUtil.isAPIAuthorizedEntityUrns( + authentication, authorizationChain, READ, List.of(urn))) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + " is unauthorized to " + READ + " entities."); + } + OperationContext opContext = + OperationContext.asSession( + systemOperationContext, + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), request, "getAspect", entityName), + authorizationChain, + authentication, + true); + + final List resultList; + if (version == 0) { + resultList = buildEntityList(opContext, List.of(urn), Set.of(aspectName), withSystemMetadata); + } else { + resultList = + buildEntityVersionedAspectList( + opContext, + new LinkedHashMap<>(Map.of(urn, Map.of(aspectName, version))), + withSystemMetadata); + } + + return resultList.stream() + .findFirst() + .flatMap( + e -> + e.getAspects().entrySet().stream() + .filter(entry -> entry.getKey().equalsIgnoreCase(aspectName)) + .map(Map.Entry::getValue) + .findFirst()) + .map(ResponseEntity::ok) + .orElse(ResponseEntity.notFound().header(NOT_FOUND_HEADER, "ENTITY").build()); + } + + @Tag(name = "Generic Aspects") + @RequestMapping( + value = "/{entityName}/{entityUrn:urn:li:.+}/{aspectName}", + method = {RequestMethod.HEAD}) + @Operation(summary = "Whether an entity aspect exists.") + public ResponseEntity headAspect( + HttpServletRequest request, + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn, + @PathVariable("aspectName") String aspectName, + @PathVariable(value = "includeSoftDelete", required = false) Boolean includeSoftDelete) + throws InvalidUrnException { + + Urn urn = validatedUrn(entityUrn); + Authentication authentication = AuthenticationContext.getAuthentication(); + if (!AuthUtil.isAPIAuthorizedEntityUrns( + authentication, authorizationChain, EXISTS, List.of(urn))) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + " is unauthorized to " + EXISTS + " entities."); + } + OperationContext opContext = + OperationContext.asSession( + systemOperationContext, + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), request, "headAspect", entityName), + authorizationChain, + authentication, + true); + + return exists(opContext, urn, lookupAspectSpec(urn, aspectName).getName(), includeSoftDelete) + ? ResponseEntity.noContent().build() + : ResponseEntity.notFound().build(); + } + + @Tag(name = "Generic Entities") + @DeleteMapping(value = "/{entityName}/{entityUrn:urn:li:.+}") + @Operation(summary = "Delete an entity") + public void deleteEntity( + HttpServletRequest request, + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn) + throws InvalidUrnException { + + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); + Urn urn = validatedUrn(entityUrn); + Authentication authentication = AuthenticationContext.getAuthentication(); + if (!AuthUtil.isAPIAuthorizedEntityUrns( + authentication, authorizationChain, DELETE, List.of(urn))) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + " is unauthorized to " + DELETE + " entities."); + } + OperationContext opContext = + OperationContext.asSession( + systemOperationContext, + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), request, "deleteEntity", entityName), + authorizationChain, + authentication, + true); + + entityService.deleteUrn(opContext, urn); + } + + @Tag(name = "Generic Entities") + @PostMapping(value = "/{entityName}", produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Create a batch of entities.") + public ResponseEntity> createEntity( + HttpServletRequest request, + @PathVariable("entityName") String entityName, + @RequestParam(value = "async", required = false, defaultValue = "true") Boolean async, + @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") + Boolean withSystemMetadata, + @RequestBody @Nonnull String jsonEntityList) + throws InvalidUrnException, JsonProcessingException { + + Authentication authentication = AuthenticationContext.getAuthentication(); + + if (!AuthUtil.isAPIAuthorizedEntityType( + authentication, authorizationChain, CREATE, entityName)) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + " is unauthorized to " + CREATE + " entities."); + } + + OperationContext opContext = + OperationContext.asSession( + systemOperationContext, + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), request, "createEntity", entityName), + authorizationChain, + authentication, + true); + + AspectsBatch batch = toMCPBatch(opContext, jsonEntityList, authentication.getActor()); + Set results = entityService.ingestProposal(opContext, batch, async); + + if (!async) { + return ResponseEntity.ok(buildEntityList(results, withSystemMetadata)); + } else { + return ResponseEntity.accepted().body(buildEntityList(results, withSystemMetadata)); + } + } + + @Tag(name = "Generic Aspects") + @DeleteMapping(value = "/{entityName}/{entityUrn:urn:li:.+}/{aspectName}") + @Operation(summary = "Delete an entity aspect.") + public void deleteAspect( + HttpServletRequest request, + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn, + @PathVariable("aspectName") String aspectName) + throws InvalidUrnException { + + Urn urn = validatedUrn(entityUrn); + Authentication authentication = AuthenticationContext.getAuthentication(); + if (!AuthUtil.isAPIAuthorizedEntityUrns( + authentication, authorizationChain, DELETE, List.of(urn))) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + " is unauthorized to " + DELETE + " entities."); + } + OperationContext opContext = + OperationContext.asSession( + systemOperationContext, + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), request, "deleteAspect", entityName), + authorizationChain, + authentication, + true); + + entityService.deleteAspect( + opContext, entityUrn, lookupAspectSpec(urn, aspectName).getName(), Map.of(), true); + } + + @Tag(name = "Generic Aspects") + @PostMapping( + value = "/{entityName}/{entityUrn:urn:li:.+}/{aspectName}", + produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Create an entity aspect.") + public ResponseEntity createAspect( + HttpServletRequest request, + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn, + @PathVariable("aspectName") String aspectName, + @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") + Boolean withSystemMetadata, + @RequestParam(value = "createIfNotExists", required = false, defaultValue = "false") + Boolean createIfNotExists, + @RequestBody @Nonnull String jsonAspect) + throws URISyntaxException { + + Urn urn = validatedUrn(entityUrn); + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); + Authentication authentication = AuthenticationContext.getAuthentication(); + + if (!AuthUtil.isAPIAuthorizedEntityUrns( + authentication, authorizationChain, CREATE, List.of(urn))) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + " is unauthorized to " + CREATE + " entities."); + } + OperationContext opContext = + OperationContext.asSession( + systemOperationContext, + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), request, "createAspect", entityName), + authorizationChain, + authentication, + true); + + AspectSpec aspectSpec = lookupAspectSpec(entitySpec, aspectName); + ChangeMCP upsert = + toUpsertItem( + opContext.getRetrieverContext().get().getAspectRetriever(), + urn, + aspectSpec, + createIfNotExists, + jsonAspect, + authentication.getActor()); + + List results = + entityService.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(upsert)) + .build(), + true, + true); + + return ResponseEntity.of( + results.stream() + .findFirst() + .map(result -> buildGenericEntity(aspectName, result, withSystemMetadata))); + } + + @Tag(name = "Generic Aspects") + @PatchMapping( + value = "/{entityName}/{entityUrn:urn:li:.+}/{aspectName}", + consumes = {"application/json-patch+json", MediaType.APPLICATION_JSON_VALUE}, + produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Patch an entity aspect. (Experimental)") + public ResponseEntity patchAspect( + HttpServletRequest request, + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn, + @PathVariable("aspectName") String aspectName, + @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") + Boolean withSystemMetadata, + @RequestBody @Nonnull GenericJsonPatch patch) + throws InvalidUrnException, + NoSuchMethodException, + InvocationTargetException, + InstantiationException, + IllegalAccessException { + + Urn urn = validatedUrn(entityUrn); + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); + Authentication authentication = AuthenticationContext.getAuthentication(); + if (!AuthUtil.isAPIAuthorizedEntityUrns( + authentication, authorizationChain, UPDATE, List.of(urn))) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + " is unauthorized to " + UPDATE + " entities."); + } + OperationContext opContext = + OperationContext.asSession( + systemOperationContext, + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), request, "patchAspect", entityName), + authorizationChain, + authentication, + true); + + AspectSpec aspectSpec = lookupAspectSpec(entitySpec, aspectName); + RecordTemplate currentValue = entityService.getAspect(opContext, urn, aspectSpec.getName(), 0); + + GenericPatchTemplate genericPatchTemplate = + GenericPatchTemplate.builder() + .genericJsonPatch(patch) + .templateType(aspectSpec.getDataTemplateClass()) + .templateDefault( + aspectSpec.getDataTemplateClass().getDeclaredConstructor().newInstance()) + .build(); + ChangeMCP upsert = + toUpsertItem( + opContext.getRetrieverContext().get().getAspectRetriever(), + validatedUrn(entityUrn), + aspectSpec, + currentValue, + genericPatchTemplate, + authentication.getActor()); + + List results = + entityService.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(upsert)) + .build(), + true, + true); + + return results.stream() + .findFirst() + .map(result -> buildGenericEntity(aspectSpec.getName(), result, withSystemMetadata)) + .map(ResponseEntity::ok) + .orElse(ResponseEntity.notFound().header(NOT_FOUND_HEADER, "ENTITY").build()); + } + + protected Boolean exists( + @Nonnull OperationContext opContext, + Urn urn, + @Nullable String aspect, + @Nullable Boolean includeSoftDelete) { + return aspect == null + ? entityService.exists( + opContext, urn, includeSoftDelete != null ? includeSoftDelete : false) + : entityService.exists( + opContext, urn, aspect, includeSoftDelete != null ? includeSoftDelete : false); + } + + /** + * Given a map with aspect names from the API, normalized them into actual aspect names (casing + * fixes) + * + * @param requestedAspectNames requested aspects + * @return updated map + * @param map values + */ + protected LinkedHashMap> resolveAspectNames( + LinkedHashMap> requestedAspectNames, @Nonnull T defaultValue) { + return requestedAspectNames.entrySet().stream() + .map( + entry -> { + final Urn urn = entry.getKey(); + if (entry.getValue().isEmpty() || entry.getValue().containsKey("")) { + // All aspects specified + Set allNames = + entityRegistry.getEntitySpec(urn.getEntityType()).getAspectSpecs().stream() + .map(AspectSpec::getName) + .collect(Collectors.toSet()); + return Map.entry( + urn, + allNames.stream() + .map( + aspectName -> + Map.entry( + aspectName, entry.getValue().getOrDefault("", defaultValue))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + } else { + final Map normalizedNames = + entry.getValue().keySet().stream() + .map( + requestAspectName -> + Map.entry( + requestAspectName, + lookupAspectSpec(urn, requestAspectName).getName())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + return Map.entry( + urn, + entry.getValue().entrySet().stream() + .filter(reqEntry -> normalizedNames.containsKey(reqEntry.getKey())) + .map( + reqEntry -> + Map.entry( + normalizedNames.get(reqEntry.getKey()), reqEntry.getValue())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + } + }) + .collect( + Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue, + (a, b) -> { + throw new IllegalStateException("Duplicate key"); + }, + LinkedHashMap::new)); + } + + protected Map> toAspectMap( + Urn urn, List aspects, boolean withSystemMetadata) { + return aspects.stream() + .map( + a -> + Map.entry( + a.getName(), + Pair.of( + toRecordTemplate(lookupAspectSpec(urn, a.getName()), a), + withSystemMetadata ? a.getSystemMetadata() : null))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + protected AspectSpec lookupAspectSpec(Urn urn, String aspectName) { + return lookupAspectSpec(entityRegistry.getEntitySpec(urn.getEntityType()), aspectName); + } + + protected RecordTemplate toRecordTemplate( + AspectSpec aspectSpec, EnvelopedAspect envelopedAspect) { + return RecordUtils.toRecordTemplate( + aspectSpec.getDataTemplateClass(), envelopedAspect.getValue().data()); + } + + protected ChangeMCP toUpsertItem( + @Nonnull AspectRetriever aspectRetriever, + Urn entityUrn, + AspectSpec aspectSpec, + Boolean createIfNotExists, + String jsonAspect, + Actor actor) + throws URISyntaxException { + return ChangeItemImpl.builder() + .urn(entityUrn) + .aspectName(aspectSpec.getName()) + .changeType(Boolean.TRUE.equals(createIfNotExists) ? ChangeType.CREATE : ChangeType.UPSERT) + .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr())) + .recordTemplate( + GenericRecordUtils.deserializeAspect( + ByteString.copyString(jsonAspect, StandardCharsets.UTF_8), + GenericRecordUtils.JSON, + aspectSpec)) + .build(aspectRetriever); + } + + protected ChangeMCP toUpsertItem( + @Nonnull AspectRetriever aspectRetriever, + @Nonnull Urn urn, + @Nonnull AspectSpec aspectSpec, + @Nullable RecordTemplate currentValue, + @Nonnull GenericPatchTemplate genericPatchTemplate, + @Nonnull Actor actor) { + return ChangeItemImpl.fromPatch( + urn, + aspectSpec, + currentValue, + genericPatchTemplate, + AuditStampUtils.createAuditStamp(actor.toUrnStr()), + aspectRetriever); + } + + /** + * Case-insensitive fallback + * + * @return + */ + protected static AspectSpec lookupAspectSpec(EntitySpec entitySpec, String aspectName) { + return entitySpec.getAspectSpec(aspectName) != null + ? entitySpec.getAspectSpec(aspectName) + : entitySpec.getAspectSpecs().stream() + .filter(aspec -> aspec.getName().toLowerCase().equals(aspectName)) + .findFirst() + .get(); + } + + protected static Urn validatedUrn(String urn) throws InvalidUrnException { + try { + return Urn.createFromString(urn); + } catch (URISyntaxException e) { + throw new InvalidUrnException(urn, "Invalid urn!"); + } + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/openlineage/controller/LineageApiImpl.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/openlineage/controller/LineageApiImpl.java index f483d6ded1f10..f1da20fefe6bf 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/openlineage/controller/LineageApiImpl.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/openlineage/controller/LineageApiImpl.java @@ -14,6 +14,7 @@ import io.datahubproject.openlineage.generated.controller.LineageApi; import io.openlineage.client.OpenLineage; import io.openlineage.client.OpenLineageClientUtils; +import jakarta.servlet.http.HttpServletRequest; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -50,6 +51,8 @@ public Optional getObjectMapper() { return Optional.of(OBJECT_MAPPER); } + @Autowired private HttpServletRequest request; + @Override public ResponseEntity postRunEventRaw(String body) { try { @@ -68,7 +71,9 @@ public ResponseEntity postRunEventRaw(OpenLineage.RunEvent openlineageRunE OperationContext opContext = OperationContext.asSession( systemOperationContext, - RequestContext.builder().buildOpenapi("postRunEventRaw", List.of()), + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), request, "postRunEventRaw", List.of()), _authorizerChain, authentication, true); diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/elastic/OperationsController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/elastic/OperationsController.java index 1718beeaeaba3..ddbc8004081eb 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/elastic/OperationsController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/elastic/OperationsController.java @@ -3,7 +3,16 @@ import com.datahub.authentication.Authentication; import com.datahub.authentication.AuthenticationContext; import com.datahub.authorization.AuthUtil; -import com.datahub.plugins.auth.authorization.Authorizer; +import com.datahub.authorization.AuthorizerChain; +import com.deblock.jsondiff.DiffGenerator; +import com.deblock.jsondiff.matcher.CompositeJsonMatcher; +import com.deblock.jsondiff.matcher.LenientJsonArrayPartialMatcher; +import com.deblock.jsondiff.matcher.LenientJsonObjectPartialMatcher; +import com.deblock.jsondiff.matcher.LenientNumberPrimitivePartialMatcher; +import com.deblock.jsondiff.matcher.StrictPrimitivePartialMatcher; +import com.deblock.jsondiff.viewer.PatchDiffViewer; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.common.urn.UrnUtils; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.entity.EntityService; @@ -11,7 +20,6 @@ import com.linkedin.metadata.entity.restoreindices.RestoreIndicesResult; import com.linkedin.metadata.query.SearchFlags; import com.linkedin.metadata.query.filter.Filter; -import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.systemmetadata.SystemMetadataService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; @@ -23,7 +31,10 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Optional; import java.util.Set; @@ -54,25 +65,29 @@ name = "ElasticSearchOperations", description = "An API for managing your elasticsearch instance") public class OperationsController { - private final Authorizer authorizerChain; + private final AuthorizerChain authorizerChain; private final OperationContext systemOperationContext; private final SystemMetadataService systemMetadataService; private final TimeseriesAspectService timeseriesAspectService; private final EntitySearchService searchService; private final EntityService entityService; + private final ObjectMapper objectMapper; public OperationsController( OperationContext systemOperationContext, SystemMetadataService systemMetadataService, TimeseriesAspectService timeseriesAspectService, EntitySearchService searchService, - EntityService entityService) { + EntityService entityService, + AuthorizerChain authorizerChain, + ObjectMapper objectMapper) { this.systemOperationContext = systemOperationContext; - this.authorizerChain = systemOperationContext.getAuthorizerContext().getAuthorizer(); + this.authorizerChain = authorizerChain; this.systemMetadataService = systemMetadataService; this.timeseriesAspectService = timeseriesAspectService; this.searchService = searchService; this.entityService = entityService; + this.objectMapper = objectMapper; } @InitBinder @@ -119,7 +134,7 @@ public ResponseEntity getTaskStatus(String task) { @Tag(name = "ElasticSearchOperations") @GetMapping(path = "/getIndexSizes", produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Get Index Sizes") - public ResponseEntity getIndexSizes() { + public ResponseEntity getIndexSizes(HttpServletRequest request) { Authentication authentication = AuthenticationContext.getAuthentication(); String actorUrnStr = authentication.getActor().toUrnStr(); @@ -131,7 +146,7 @@ public ResponseEntity getIndexSizes() { OperationContext opContext = OperationContext.asSession( systemOperationContext, - RequestContext.builder().buildOpenapi("getIndexSizes", List.of()), + RequestContext.builder().buildOpenapi(actorUrnStr, request, "getIndexSizes", List.of()), authorizerChain, authentication, true); @@ -157,12 +172,13 @@ public ResponseEntity getIndexSizes() { @GetMapping(path = "/explainSearchQuery", produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Explain Search Query") public ResponseEntity explainSearchQuery( + HttpServletRequest request, @Parameter( name = "query", required = true, description = "Query to evaluate for specified document, will be applied as an input string to standard search query builder.") - @RequestParam("query") + @RequestParam(value = "query", defaultValue = "*") @Nonnull String query, @Parameter( @@ -176,16 +192,15 @@ public ResponseEntity explainSearchQuery( name = "entityName", required = true, description = "Name of the entity the document belongs to.") - @RequestParam("entityName") + @RequestParam(value = "entityName", defaultValue = "dataset") @Nonnull String entityName, - @Parameter(name = "scrollId", required = false, description = "Scroll ID for pagination.") + @Parameter(name = "scrollId", description = "Scroll ID for pagination.") @RequestParam("scrollId") @Nullable String scrollId, @Parameter( name = "keepAlive", - required = false, description = "Keep alive time for point in time scroll context" + ", only relevant where point in time is supported.") @@ -193,73 +208,204 @@ public ResponseEntity explainSearchQuery( @Nullable String keepAlive, @Parameter(name = "size", required = true, description = "Page size for pagination.") - @RequestParam("size") + @RequestParam(value = "size", required = false, defaultValue = "1") int size, - @Parameter( - name = "filters", - required = false, - description = "Additional filters to apply to query.") - @RequestParam("filters") + @Parameter(name = "filters", description = "Additional filters to apply to query.") + @RequestParam(value = "filters", required = false) @Nullable - Filter filters, - @Parameter( - name = "sortCriterion", - required = false, - description = "Criterion to sort results on.") - @RequestParam("sortCriterion") + String filters, + @Parameter(name = "searchFlags", description = "Optional configuration flags.") + @RequestParam(value = "searchFlags", required = false) @Nullable - SortCriterion sortCriterion, + String searchFlags) + throws JsonProcessingException { + + Authentication authentication = AuthenticationContext.getAuthentication(); + String actorUrnStr = authentication.getActor().toUrnStr(); + + if (!AuthUtil.isAPIAuthorized( + authentication, authorizerChain, PoliciesConfig.ES_EXPLAIN_QUERY_PRIVILEGE)) { + log.error("{} is not authorized to get explain queries", actorUrnStr); + return ResponseEntity.status(HttpStatus.FORBIDDEN).body(null); + } + OperationContext opContext = + systemOperationContext + .asSession( + RequestContext.builder() + .buildOpenapi(actorUrnStr, request, "explainSearchQuery", entityName), + authorizerChain, + authentication) + .withSearchFlags( + flags -> { + try { + return searchFlags == null + ? flags + : objectMapper.readValue(searchFlags, SearchFlags.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); + + ExplainResponse response = + searchService.explain( + opContext, + query, + encodeValue(documentId), + entityName, + filters == null ? null : objectMapper.readValue(filters, Filter.class), + null, + scrollId, + keepAlive, + size, + null); + + return ResponseEntity.ok(response); + } + + @Tag(name = "ElasticSearchOperations") + @GetMapping(path = "/explainSearchQueryDiff", produces = MediaType.TEXT_PLAIN_VALUE) + @Operation(summary = "Explain the differences in scoring for 2 documents") + public ResponseEntity explainSearchQueryDiff( + HttpServletRequest request, @Parameter( - name = "searchFlags", - required = false, - description = "Optional configuration flags.") - @RequestParam("searchFlags") + name = "query", + required = true, + description = + "Query to evaluate for specified document, will be applied as an input string to standard search query builder.") + @RequestParam(value = "query", defaultValue = "*") + @Nonnull + String query, + @Parameter( + name = "documentIdA", + required = true, + description = "Document 1st ID to apply explain to.") + @RequestParam("documentIdA") + @Nonnull + String documentIdA, + @Parameter( + name = "documentIdB", + required = true, + description = "Document 2nd ID to apply explain to.") + @RequestParam("documentIdB") + @Nonnull + String documentIdB, + @Parameter( + name = "entityName", + required = true, + description = "Name of the entity the document belongs to.") + @RequestParam(value = "entityName", defaultValue = "dataset") + @Nonnull + String entityName, + @Parameter(name = "scrollId", description = "Scroll ID for pagination.") + @RequestParam("scrollId") @Nullable - SearchFlags searchFlags, + String scrollId, @Parameter( - name = "facets", - required = false, - description = "List of facet fields for aggregations.") - @RequestParam("facets") + name = "keepAlive", + description = + "Keep alive time for point in time scroll context" + + ", only relevant where point in time is supported.") + @RequestParam("keepAlive") @Nullable - List facets) { + String keepAlive, + @Parameter(name = "size", required = true, description = "Page size for pagination.") + @RequestParam(value = "size", required = false, defaultValue = "1") + int size, + @Parameter(name = "filters", description = "Additional filters to apply to query.") + @RequestParam(value = "filters", required = false) + @Nullable + String filters, + @Parameter(name = "searchFlags", description = "Optional configuration flags.") + @RequestParam(value = "searchFlags", required = false) + @Nullable + String searchFlags) + throws JsonProcessingException { Authentication authentication = AuthenticationContext.getAuthentication(); String actorUrnStr = authentication.getActor().toUrnStr(); if (!AuthUtil.isAPIAuthorized( authentication, authorizerChain, PoliciesConfig.ES_EXPLAIN_QUERY_PRIVILEGE)) { - log.error("{} is not authorized to get timeseries index sizes", actorUrnStr); + log.error("{} is not authorized to get explain queries", actorUrnStr); return ResponseEntity.status(HttpStatus.FORBIDDEN).body(null); } OperationContext opContext = systemOperationContext .asSession( - RequestContext.builder().buildOpenapi("explainSearchQuery", entityName), + RequestContext.builder() + .buildOpenapi(actorUrnStr, request, "explainSearchQuery", entityName), authorizerChain, authentication) - .withSearchFlags(flags -> searchFlags); + .withSearchFlags( + flags -> { + try { + return searchFlags == null + ? flags + : objectMapper.readValue(searchFlags, SearchFlags.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); - ExplainResponse response = + ExplainResponse responseA = searchService.explain( opContext, query, - documentId, + encodeValue(documentIdA), entityName, - filters, - sortCriterion, + filters == null ? null : objectMapper.readValue(filters, Filter.class), + null, scrollId, keepAlive, size, - facets); + null); - return ResponseEntity.ok(response); + ExplainResponse responseB = + searchService.explain( + opContext, + query, + encodeValue(documentIdB), + entityName, + filters == null ? null : objectMapper.readValue(filters, Filter.class), + null, + scrollId, + keepAlive, + size, + null); + + String a = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(responseA); + String b = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(responseB); + + CompositeJsonMatcher fullLenient = + new CompositeJsonMatcher( + new LenientJsonArrayPartialMatcher(), // comparing array using lenient mode (ignore + // array order and extra items) + new LenientJsonObjectPartialMatcher(), // comparing object using lenient mode (ignoring + // extra properties) + new LenientNumberPrimitivePartialMatcher( + new StrictPrimitivePartialMatcher()) // comparing primitive types and manage numbers + // (100.00 == 100) + ); + + // generate a diff + final var jsondiff = DiffGenerator.diff(a, b, fullLenient); + PatchDiffViewer patch = PatchDiffViewer.from(jsondiff); + + return ResponseEntity.ok(patch.toString()); + } + + private static String encodeValue(String value) { + if (value.startsWith("urn:li:")) { + return URLEncoder.encode(value, StandardCharsets.UTF_8); + } + return value; } @Tag(name = "RestoreIndices") @GetMapping(path = "/restoreIndices", produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Restore ElasticSearch indices from primary storage based on URNs.") public ResponseEntity> restoreIndices( + HttpServletRequest request, @RequestParam(required = false, name = "aspectName") @Nullable String aspectName, @RequestParam(required = false, name = "urn") @Nullable String urn, @RequestParam(required = false, name = "urnLike") @Nullable String urnLike, @@ -279,7 +425,9 @@ public ResponseEntity> restoreIndices( OperationContext opContext = OperationContext.asSession( systemOperationContext, - RequestContext.builder().buildOpenapi("restoreIndices", List.of()), + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), request, "restoreIndices", List.of()), authorizerChain, authentication, true); @@ -305,6 +453,7 @@ public ResponseEntity> restoreIndices( @PostMapping(path = "/restoreIndices", produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Restore ElasticSearch indices from primary storage based on URNs.") public ResponseEntity> restoreIndices( + HttpServletRequest request, @RequestParam(required = false, name = "aspectNames") @Nullable Set aspectNames, @RequestParam(required = false, name = "batchSize", defaultValue = "100") @Nullable Integer batchSize, @@ -319,7 +468,9 @@ public ResponseEntity> restoreIndices( OperationContext opContext = OperationContext.asSession( systemOperationContext, - RequestContext.builder().buildOpenapi("restoreIndices", List.of()), + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), request, "restoreIndices", List.of()), authorizerChain, authentication, true); diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v1/entities/EntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v1/entities/EntitiesController.java index 4ecb91f193857..99eede15629d2 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v1/entities/EntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v1/entities/EntitiesController.java @@ -30,6 +30,7 @@ import io.datahubproject.openapi.util.MappingUtil; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; import java.net.URLDecoder; import java.util.Arrays; import java.util.Collections; @@ -55,6 +56,10 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +/* + Use v2 or v3 controllers instead +*/ +@Deprecated @RestController @RequestMapping("/entities/v1") @Slf4j @@ -86,6 +91,7 @@ public void initBinder(WebDataBinder binder) { @GetMapping(value = "/latest", produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity getEntities( + HttpServletRequest request, @Parameter( name = "urns", required = true, @@ -118,6 +124,8 @@ public ResponseEntity getEntities( systemOperationContext, RequestContext.builder() .buildOpenapi( + actorUrnStr, + request, "getEntities", entityUrns.stream() .map(Urn::getEntityType) @@ -165,6 +173,7 @@ public ResponseEntity getEntities( @PostMapping(value = "/", produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity> postEntities( + HttpServletRequest request, @RequestBody @Nonnull List aspectRequests, @RequestParam(required = false, name = "async") Boolean async, @RequestParam(required = false, name = "createIfNotExists") Boolean createIfNotExists, @@ -187,6 +196,8 @@ public ResponseEntity> postEntities( systemOperationContext, RequestContext.builder() .buildOpenapi( + actorUrnStr, + request, "postEntities", proposals.stream() .map(MetadataChangeProposal::getEntityType) @@ -237,6 +248,7 @@ public ResponseEntity> postEntities( @DeleteMapping(value = "/", produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity> deleteEntities( + HttpServletRequest request, @Parameter( name = "urns", required = true, @@ -274,6 +286,8 @@ public ResponseEntity> deleteEntities( systemOperationContext, RequestContext.builder() .buildOpenapi( + actorUrnStr, + request, "deleteEntities", entityUrns.stream().map(Urn::getEntityType).collect(Collectors.toSet())), _authorizerChain, diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v1/relationships/RelationshipsController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v1/relationships/RelationshipsController.java index 8460e3a57abf5..f91ebb61123f0 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v1/relationships/RelationshipsController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v1/relationships/RelationshipsController.java @@ -41,6 +41,10 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +/* + Use v2 or v3 controllers instead +*/ +@Deprecated @RestController @RequiredArgsConstructor @RequestMapping("/relationships/v1") diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v1/timeline/TimelineControllerV1.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v1/timeline/TimelineControllerV1.java new file mode 100644 index 0000000000000..9456843a3d810 --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v1/timeline/TimelineControllerV1.java @@ -0,0 +1,89 @@ +package io.datahubproject.openapi.v1.timeline; + +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; +import com.datahub.authorization.AuthUtil; +import com.datahub.authorization.AuthorizerChain; +import com.datahub.authorization.ConjunctivePrivilegeGroup; +import com.datahub.authorization.DisjunctivePrivilegeGroup; +import com.datahub.authorization.EntitySpec; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.authorization.PoliciesConfig; +import com.linkedin.metadata.timeline.TimelineService; +import com.linkedin.metadata.timeline.data.ChangeCategory; +import com.linkedin.metadata.timeline.data.ChangeTransaction; +import io.datahubproject.openapi.exception.UnauthorizedException; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Set; +import lombok.AllArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +/* + Use v2 or v3 controllers instead +*/ +@Deprecated +@RestController +@AllArgsConstructor +@RequestMapping("/timeline/v1") +@Tag( + name = "Timeline", + description = + "An API for retrieving historical updates to entities and their related documentation.") +public class TimelineControllerV1 { + + private final TimelineService _timelineService; + private final AuthorizerChain _authorizerChain; + + @Value("${authorization.restApiAuthorization:false}") + private Boolean restApiAuthorizationEnabled; + + /** + * @param rawUrn + * @param startTime + * @param endTime + * @param raw + * @param categories + * @return + * @throws URISyntaxException + * @throws JsonProcessingException + */ + @GetMapping(path = "/{urn}", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity> getTimeline( + @PathVariable("urn") String rawUrn, + @RequestParam(name = "startTime", defaultValue = "-1") long startTime, + @RequestParam(name = "endTime", defaultValue = "0") long endTime, + @RequestParam(name = "raw", defaultValue = "false") boolean raw, + @RequestParam(name = "categories") Set categories) + throws URISyntaxException, JsonProcessingException { + // Make request params when implemented + String startVersionStamp = null; + String endVersionStamp = null; + Urn urn = Urn.createFromString(rawUrn); + Authentication authentication = AuthenticationContext.getAuthentication(); + String actorUrnStr = authentication.getActor().toUrnStr(); + EntitySpec resourceSpec = new EntitySpec(urn.getEntityType(), rawUrn); + DisjunctivePrivilegeGroup orGroup = + new DisjunctivePrivilegeGroup( + ImmutableList.of( + new ConjunctivePrivilegeGroup( + ImmutableList.of(PoliciesConfig.GET_TIMELINE_PRIVILEGE.getType())))); + if (restApiAuthorizationEnabled + && !AuthUtil.isAuthorized(_authorizerChain, actorUrnStr, orGroup, resourceSpec)) { + throw new UnauthorizedException(actorUrnStr + " is unauthorized to edit entities."); + } + return ResponseEntity.ok( + _timelineService.getTimeline( + urn, categories, startTime, endTime, startVersionStamp, endVersionStamp, raw)); + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java index 41cf972079c25..54a7724cadd34 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java @@ -1,175 +1,84 @@ package io.datahubproject.openapi.v2.controller; -import static com.linkedin.metadata.authorization.ApiOperation.CREATE; -import static com.linkedin.metadata.authorization.ApiOperation.DELETE; -import static com.linkedin.metadata.authorization.ApiOperation.EXISTS; import static com.linkedin.metadata.authorization.ApiOperation.READ; -import static com.linkedin.metadata.authorization.ApiOperation.UPDATE; import com.datahub.authentication.Actor; import com.datahub.authentication.Authentication; import com.datahub.authentication.AuthenticationContext; import com.datahub.authorization.AuthUtil; -import com.datahub.authorization.AuthorizerChain; -import com.datahub.util.RecordUtils; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.ByteString; import com.linkedin.data.template.RecordTemplate; import com.linkedin.entity.EnvelopedAspect; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.batch.AspectsBatch; import com.linkedin.metadata.aspect.batch.BatchItem; -import com.linkedin.metadata.aspect.batch.ChangeMCP; -import com.linkedin.metadata.aspect.patch.GenericJsonPatch; -import com.linkedin.metadata.aspect.patch.template.common.GenericPatchTemplate; import com.linkedin.metadata.entity.EntityApiUtils; -import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.IngestResult; import com.linkedin.metadata.entity.UpdateAspectResult; import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl; import com.linkedin.metadata.models.AspectSpec; -import com.linkedin.metadata.models.EntitySpec; -import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.query.SearchFlags; -import com.linkedin.metadata.query.filter.SortCriterion; -import com.linkedin.metadata.query.filter.SortOrder; -import com.linkedin.metadata.search.ScrollResult; import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchEntityArray; -import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.metadata.utils.GenericRecordUtils; -import com.linkedin.metadata.utils.SearchUtil; import com.linkedin.mxe.SystemMetadata; import com.linkedin.util.Pair; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.metadata.context.RequestContext; +import io.datahubproject.openapi.controller.GenericEntitiesController; +import io.datahubproject.openapi.exception.InvalidUrnException; import io.datahubproject.openapi.exception.UnauthorizedException; -import io.datahubproject.openapi.v2.models.BatchGetUrnRequest; -import io.datahubproject.openapi.v2.models.BatchGetUrnResponse; -import io.datahubproject.openapi.v2.models.GenericEntity; -import io.datahubproject.openapi.v2.models.GenericScrollResult; +import io.datahubproject.openapi.v2.models.BatchGetUrnRequestV2; +import io.datahubproject.openapi.v2.models.BatchGetUrnResponseV2; +import io.datahubproject.openapi.v2.models.GenericAspectV2; +import io.datahubproject.openapi.v2.models.GenericEntityScrollResultV2; +import io.datahubproject.openapi.v2.models.GenericEntityV2; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; -import java.lang.reflect.InvocationTargetException; +import jakarta.servlet.http.HttpServletRequest; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.DeleteMapping; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PatchMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; @RestController @RequiredArgsConstructor @RequestMapping("/v2/entity") @Slf4j -public class EntityController { - private static final SearchFlags DEFAULT_SEARCH_FLAGS = - new SearchFlags().setFulltext(false).setSkipAggregates(true).setSkipHighlighting(true); - @Autowired private EntityRegistry entityRegistry; - @Autowired private SearchService searchService; - @Autowired private EntityService entityService; - @Autowired private AuthorizerChain authorizationChain; - @Autowired private ObjectMapper objectMapper; - - @Qualifier("systemOperationContext") - @Autowired - private OperationContext systemOperationContext; - - @Tag(name = "Generic Entities", description = "API for interacting with generic entities.") - @GetMapping(value = "/{entityName}", produces = MediaType.APPLICATION_JSON_VALUE) - @Operation(summary = "Scroll entities") - public ResponseEntity> getEntities( - @PathVariable("entityName") String entityName, - @RequestParam(value = "aspectNames", defaultValue = "") Set aspectNames, - @RequestParam(value = "count", defaultValue = "10") Integer count, - @RequestParam(value = "query", defaultValue = "*") String query, - @RequestParam(value = "scrollId", required = false) String scrollId, - @RequestParam(value = "sort", required = false, defaultValue = "urn") String sortField, - @RequestParam(value = "sortOrder", required = false, defaultValue = "ASCENDING") - String sortOrder, - @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") - Boolean withSystemMetadata) - throws URISyntaxException { - - EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); - Authentication authentication = AuthenticationContext.getAuthentication(); - - if (!AuthUtil.isAPIAuthorizedEntityType(authentication, authorizationChain, READ, entityName)) { - throw new UnauthorizedException( - authentication.getActor().toUrnStr() + " is unauthorized to " + READ + " entities."); - } - - OperationContext opContext = - OperationContext.asSession( - systemOperationContext, - RequestContext.builder().buildOpenapi("getEntities", entityName), - authorizationChain, - authentication, - true); - - // TODO: support additional and multiple sort params - SortCriterion sortCriterion = SearchUtil.sortBy(sortField, SortOrder.valueOf(sortOrder)); - - ScrollResult result = - searchService.scrollAcrossEntities( - opContext.withSearchFlags(flags -> DEFAULT_SEARCH_FLAGS), - List.of(entitySpec.getName()), - query, - null, - sortCriterion, - scrollId, - null, - count); - - if (!AuthUtil.isAPIAuthorizedResult(authentication, authorizationChain, result)) { - throw new UnauthorizedException( - authentication.getActor().toUrnStr() + " is unauthorized to " + READ + " entities."); - } - - return ResponseEntity.ok( - GenericScrollResult.builder() - .results( - toRecordTemplates(opContext, result.getEntities(), aspectNames, withSystemMetadata)) - .scrollId(result.getScrollId()) - .build()); - } +public class EntityController + extends GenericEntitiesController< + GenericAspectV2, GenericEntityV2, GenericEntityScrollResultV2> { @Tag(name = "Generic Entities") @PostMapping(value = "/batch/{entityName}", produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Get a batch of entities") - public ResponseEntity getEntityBatch( - @PathVariable("entityName") String entityName, @RequestBody BatchGetUrnRequest request) + public ResponseEntity> getEntityBatch( + HttpServletRequest httpServletRequest, + @PathVariable("entityName") String entityName, + @RequestBody BatchGetUrnRequestV2 request) throws URISyntaxException { List urns = request.getUrns().stream().map(UrnUtils::getUrn).collect(Collectors.toList()); @@ -182,17 +91,22 @@ public ResponseEntity getEntityBatch( OperationContext opContext = OperationContext.asSession( systemOperationContext, - RequestContext.builder().buildOpenapi("getEntityBatch", entityName), + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), + httpServletRequest, + "getEntityBatch", + entityName), authorizationChain, authentication, true); return ResponseEntity.of( Optional.of( - BatchGetUrnResponse.builder() + BatchGetUrnResponseV2.builder() .entities( new ArrayList<>( - toRecordTemplates( + buildEntityList( opContext, urns, new HashSet<>(request.getAspectNames()), @@ -200,506 +114,24 @@ public ResponseEntity getEntityBatch( .build())); } - @Tag(name = "Generic Entities") - @GetMapping( - value = "/{entityName}/{entityUrn:urn:li:.+}", - produces = MediaType.APPLICATION_JSON_VALUE) - public ResponseEntity getEntity( - @PathVariable("entityName") String entityName, - @PathVariable("entityUrn") String entityUrn, - @RequestParam(value = "aspectNames", defaultValue = "") Set aspectNames, - @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") - Boolean withSystemMetadata) - throws URISyntaxException { - - Urn urn = UrnUtils.getUrn(entityUrn); - Authentication authentication = AuthenticationContext.getAuthentication(); - if (!AuthUtil.isAPIAuthorizedEntityUrns( - authentication, authorizationChain, READ, List.of(urn))) { - throw new UnauthorizedException( - authentication.getActor().toUrnStr() + " is unauthorized to " + READ + " entities."); - } - OperationContext opContext = - OperationContext.asSession( - systemOperationContext, - RequestContext.builder().buildOpenapi("getEntity", entityName), - authorizationChain, - authentication, - true); - - return ResponseEntity.of( - toRecordTemplates(opContext, List.of(urn), aspectNames, withSystemMetadata).stream() - .findFirst()); - } - - @Tag(name = "Generic Entities") - @RequestMapping( - value = "/{entityName}/{entityUrn}", - method = {RequestMethod.HEAD}) - @Operation(summary = "Entity exists") - public ResponseEntity headEntity( - @PathVariable("entityName") String entityName, @PathVariable("entityUrn") String entityUrn) { - - Urn urn = UrnUtils.getUrn(entityUrn); - Authentication authentication = AuthenticationContext.getAuthentication(); - if (!AuthUtil.isAPIAuthorizedEntityUrns( - authentication, authorizationChain, EXISTS, List.of(urn))) { - throw new UnauthorizedException( - authentication.getActor().toUrnStr() + " is unauthorized to " + EXISTS + " entities."); - } - OperationContext opContext = - OperationContext.asSession( - systemOperationContext, - RequestContext.builder().buildOpenapi("headEntity", entityName), - authorizationChain, - authentication, - true); - - return exists(opContext, urn, null) - ? ResponseEntity.noContent().build() - : ResponseEntity.notFound().build(); - } - - @Tag(name = "Generic Aspects", description = "API for generic aspects.") - @GetMapping( - value = "/{entityName}/{entityUrn}/{aspectName}", - produces = MediaType.APPLICATION_JSON_VALUE) - @Operation(summary = "Get an entity's generic aspect.") - public ResponseEntity getAspect( - @PathVariable("entityName") String entityName, - @PathVariable("entityUrn") String entityUrn, - @PathVariable("aspectName") String aspectName, - @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") - Boolean withSystemMetadata) - throws URISyntaxException { - - Urn urn = UrnUtils.getUrn(entityUrn); - Authentication authentication = AuthenticationContext.getAuthentication(); - if (!AuthUtil.isAPIAuthorizedEntityUrns( - authentication, authorizationChain, READ, List.of(urn))) { - throw new UnauthorizedException( - authentication.getActor().toUrnStr() + " is unauthorized to " + READ + " entities."); - } - OperationContext opContext = - OperationContext.asSession( - systemOperationContext, - RequestContext.builder().buildOpenapi("getAspect", entityName), - authorizationChain, - authentication, - true); - - return ResponseEntity.of( - toRecordTemplates(opContext, List.of(urn), Set.of(aspectName), withSystemMetadata).stream() - .findFirst() - .flatMap( - e -> - e.getAspects().entrySet().stream() - .filter( - entry -> - entry.getKey().equals(lookupAspectSpec(urn, aspectName).getName())) - .map(Map.Entry::getValue) - .findFirst())); - } - - @Tag(name = "Generic Aspects") - @RequestMapping( - value = "/{entityName}/{entityUrn}/{aspectName}", - method = {RequestMethod.HEAD}) - @Operation(summary = "Whether an entity aspect exists.") - public ResponseEntity headAspect( - @PathVariable("entityName") String entityName, - @PathVariable("entityUrn") String entityUrn, - @PathVariable("aspectName") String aspectName) { - - Urn urn = UrnUtils.getUrn(entityUrn); - Authentication authentication = AuthenticationContext.getAuthentication(); - if (!AuthUtil.isAPIAuthorizedEntityUrns( - authentication, authorizationChain, EXISTS, List.of(urn))) { - throw new UnauthorizedException( - authentication.getActor().toUrnStr() + " is unauthorized to " + EXISTS + " entities."); - } - OperationContext opContext = - OperationContext.asSession( - systemOperationContext, - RequestContext.builder().buildOpenapi("headAspect", entityName), - authorizationChain, - authentication, - true); - - return exists(opContext, urn, lookupAspectSpec(urn, aspectName).getName()) - ? ResponseEntity.noContent().build() - : ResponseEntity.notFound().build(); - } - - @Tag(name = "Generic Entities") - @DeleteMapping(value = "/{entityName}/{entityUrn}") - @Operation(summary = "Delete an entity") - public void deleteEntity( - @PathVariable("entityName") String entityName, @PathVariable("entityUrn") String entityUrn) { - - EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); - Urn urn = UrnUtils.getUrn(entityUrn); - Authentication authentication = AuthenticationContext.getAuthentication(); - if (!AuthUtil.isAPIAuthorizedEntityUrns( - authentication, authorizationChain, DELETE, List.of(urn))) { - throw new UnauthorizedException( - authentication.getActor().toUrnStr() + " is unauthorized to " + DELETE + " entities."); - } - OperationContext opContext = - OperationContext.asSession( - systemOperationContext, - RequestContext.builder().buildOpenapi("deleteEntity", entityName), - authorizationChain, - authentication, - true); - - entityService.deleteAspect(opContext, entityUrn, entitySpec.getKeyAspectName(), Map.of(), true); - } - - @Tag(name = "Generic Entities") - @PostMapping(value = "/{entityName}", produces = MediaType.APPLICATION_JSON_VALUE) - @Operation(summary = "Create a batch of entities.") - public ResponseEntity> createEntity( - @PathVariable("entityName") String entityName, - @RequestParam(value = "async", required = false, defaultValue = "true") Boolean async, - @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") - Boolean withSystemMetadata, - @RequestBody @Nonnull String jsonEntityList) - throws URISyntaxException, JsonProcessingException { - - Authentication authentication = AuthenticationContext.getAuthentication(); - - if (!AuthUtil.isAPIAuthorizedEntityType( - authentication, authorizationChain, CREATE, entityName)) { - throw new UnauthorizedException( - authentication.getActor().toUrnStr() + " is unauthorized to " + CREATE + " entities."); - } - - OperationContext opContext = - OperationContext.asSession( - systemOperationContext, - RequestContext.builder().buildOpenapi("createEntity", entityName), - authorizationChain, - authentication, - true); - - AspectsBatch batch = toMCPBatch(opContext, jsonEntityList, authentication.getActor()); - Set results = entityService.ingestProposal(opContext, batch, async); - - if (!async) { - return ResponseEntity.ok(toEntityListResponse(results, withSystemMetadata)); - } else { - return ResponseEntity.accepted().body(toEntityListResponse(results, withSystemMetadata)); - } - } - - @Tag(name = "Generic Aspects") - @DeleteMapping(value = "/{entityName}/{entityUrn}/{aspectName}") - @Operation(summary = "Delete an entity aspect.") - public void deleteAspect( - @PathVariable("entityName") String entityName, - @PathVariable("entityUrn") String entityUrn, - @PathVariable("aspectName") String aspectName) { - - Urn urn = UrnUtils.getUrn(entityUrn); - Authentication authentication = AuthenticationContext.getAuthentication(); - if (!AuthUtil.isAPIAuthorizedEntityUrns( - authentication, authorizationChain, DELETE, List.of(urn))) { - throw new UnauthorizedException( - authentication.getActor().toUrnStr() + " is unauthorized to " + DELETE + " entities."); - } - OperationContext opContext = - OperationContext.asSession( - systemOperationContext, - RequestContext.builder().buildOpenapi("deleteAspect", entityName), - authorizationChain, - authentication, - true); - - entityService.deleteAspect( - opContext, entityUrn, lookupAspectSpec(urn, aspectName).getName(), Map.of(), true); - } - - @Tag(name = "Generic Aspects") - @PostMapping( - value = "/{entityName}/{entityUrn}/{aspectName}", - produces = MediaType.APPLICATION_JSON_VALUE) - @Operation(summary = "Create an entity aspect.") - public ResponseEntity createAspect( - @PathVariable("entityName") String entityName, - @PathVariable("entityUrn") String entityUrn, - @PathVariable("aspectName") String aspectName, - @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") - Boolean withSystemMetadata, - @RequestParam(value = "createIfNotExists", required = false, defaultValue = "false") - Boolean createIfNotExists, - @RequestBody @Nonnull String jsonAspect) - throws URISyntaxException { - - Urn urn = UrnUtils.getUrn(entityUrn); - EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); - Authentication authentication = AuthenticationContext.getAuthentication(); - - if (!AuthUtil.isAPIAuthorizedEntityUrns( - authentication, authorizationChain, CREATE, List.of(urn))) { - throw new UnauthorizedException( - authentication.getActor().toUrnStr() + " is unauthorized to " + CREATE + " entities."); - } - OperationContext opContext = - OperationContext.asSession( - systemOperationContext, - RequestContext.builder().buildOpenapi("createAspect", entityName), - authorizationChain, - authentication, - true); - - AspectSpec aspectSpec = lookupAspectSpec(entitySpec, aspectName); - ChangeMCP upsert = - toUpsertItem( - opContext.getRetrieverContext().get().getAspectRetriever(), - urn, - aspectSpec, - createIfNotExists, - jsonAspect, - authentication.getActor()); - - List results = - entityService.ingestAspects( - opContext, - AspectsBatchImpl.builder() - .retrieverContext(opContext.getRetrieverContext().get()) - .items(List.of(upsert)) - .build(), - true, - true); - - return ResponseEntity.of( - results.stream() - .findFirst() - .map( - result -> - GenericEntity.builder() - .urn(result.getUrn().toString()) - .build( - objectMapper, - Map.of( - aspectName, - Pair.of( - result.getNewValue(), - withSystemMetadata ? result.getNewSystemMetadata() : null))))); - } - - @Tag(name = "Generic Aspects") - @PatchMapping( - value = "/{entityName}/{entityUrn}/{aspectName}", - consumes = "application/json-patch+json", - produces = MediaType.APPLICATION_JSON_VALUE) - @Operation(summary = "Patch an entity aspect. (Experimental)") - public ResponseEntity patchAspect( - @PathVariable("entityName") String entityName, - @PathVariable("entityUrn") String entityUrn, - @PathVariable("aspectName") String aspectName, - @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") - Boolean withSystemMetadata, - @RequestBody @Nonnull GenericJsonPatch patch) - throws URISyntaxException, - NoSuchMethodException, - InvocationTargetException, - InstantiationException, - IllegalAccessException { - - Urn urn = UrnUtils.getUrn(entityUrn); - EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); - Authentication authentication = AuthenticationContext.getAuthentication(); - if (!AuthUtil.isAPIAuthorizedEntityUrns( - authentication, authorizationChain, UPDATE, List.of(urn))) { - throw new UnauthorizedException( - authentication.getActor().toUrnStr() + " is unauthorized to " + UPDATE + " entities."); - } - OperationContext opContext = - OperationContext.asSession( - systemOperationContext, - RequestContext.builder().buildOpenapi("patchAspect", entityName), - authorizationChain, - authentication, - true); - - AspectSpec aspectSpec = lookupAspectSpec(entitySpec, aspectName); - RecordTemplate currentValue = entityService.getAspect(opContext, urn, aspectSpec.getName(), 0); - - GenericPatchTemplate genericPatchTemplate = - GenericPatchTemplate.builder() - .genericJsonPatch(patch) - .templateType(aspectSpec.getDataTemplateClass()) - .templateDefault( - aspectSpec.getDataTemplateClass().getDeclaredConstructor().newInstance()) - .build(); - ChangeMCP upsert = - toUpsertItem( - opContext.getRetrieverContext().get().getAspectRetriever(), - UrnUtils.getUrn(entityUrn), - aspectSpec, - currentValue, - genericPatchTemplate, - authentication.getActor()); - - List results = - entityService.ingestAspects( - opContext, - AspectsBatchImpl.builder() - .retrieverContext(opContext.getRetrieverContext().get()) - .items(List.of(upsert)) - .build(), - true, - true); - - return ResponseEntity.of( - results.stream() - .findFirst() - .map( - result -> - GenericEntity.builder() - .urn(result.getUrn().toString()) - .build( - objectMapper, - Map.of( - aspectSpec.getName(), - Pair.of( - result.getNewValue(), - withSystemMetadata ? result.getNewSystemMetadata() : null))))); - } - - private List toRecordTemplates( + @Override + public GenericEntityScrollResultV2 buildScrollResult( @Nonnull OperationContext opContext, SearchEntityArray searchEntities, Set aspectNames, - boolean withSystemMetadata) + boolean withSystemMetadata, + @Nullable String scrollId) throws URISyntaxException { - return toRecordTemplates( - opContext, - searchEntities.stream().map(SearchEntity::getEntity).collect(Collectors.toList()), - aspectNames, - withSystemMetadata); - } - - private Boolean exists(@Nonnull OperationContext opContext, Urn urn, @Nullable String aspect) { - return aspect == null - ? entityService.exists(opContext, urn, true) - : entityService.exists(opContext, urn, aspect, true); - } - - private List toRecordTemplates( - @Nonnull OperationContext opContext, - List urns, - Set aspectNames, - boolean withSystemMetadata) - throws URISyntaxException { - if (urns.isEmpty()) { - return List.of(); - } else { - Set urnsSet = new HashSet<>(urns); - - Map> aspects = - entityService.getLatestEnvelopedAspects( - opContext, - urnsSet, - resolveAspectNames(urnsSet, aspectNames).stream() - .map(AspectSpec::getName) - .collect(Collectors.toSet())); - - return urns.stream() - .map( - u -> - GenericEntity.builder() - .urn(u.toString()) - .build( - objectMapper, - toAspectMap(u, aspects.getOrDefault(u, List.of()), withSystemMetadata))) - .collect(Collectors.toList()); - } - } - - private Set resolveAspectNames(Set urns, Set requestedAspectNames) { - if (requestedAspectNames.isEmpty()) { - return urns.stream() - .flatMap(u -> entityRegistry.getEntitySpec(u.getEntityType()).getAspectSpecs().stream()) - .collect(Collectors.toSet()); - } else { - // ensure key is always present - return Stream.concat( - urns.stream() - .flatMap( - urn -> - requestedAspectNames.stream() - .map(aspectName -> lookupAspectSpec(urn, aspectName))), - urns.stream() - .map(u -> entityRegistry.getEntitySpec(u.getEntityType()).getKeyAspectSpec())) - .collect(Collectors.toSet()); - } - } - - private Map> toAspectMap( - Urn urn, List aspects, boolean withSystemMetadata) { - return aspects.stream() - .map( - a -> - Map.entry( - a.getName(), - Pair.of( - toRecordTemplate(lookupAspectSpec(urn, a.getName()), a), - withSystemMetadata ? a.getSystemMetadata() : null))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - private AspectSpec lookupAspectSpec(Urn urn, String aspectName) { - return lookupAspectSpec(entityRegistry.getEntitySpec(urn.getEntityType()), aspectName); - } - - private RecordTemplate toRecordTemplate(AspectSpec aspectSpec, EnvelopedAspect envelopedAspect) { - return RecordUtils.toRecordTemplate( - aspectSpec.getDataTemplateClass(), envelopedAspect.getValue().data()); - } - - private ChangeMCP toUpsertItem( - @Nonnull AspectRetriever aspectRetriever, - Urn entityUrn, - AspectSpec aspectSpec, - Boolean createIfNotExists, - String jsonAspect, - Actor actor) - throws URISyntaxException { - return ChangeItemImpl.builder() - .urn(entityUrn) - .aspectName(aspectSpec.getName()) - .changeType(Boolean.TRUE.equals(createIfNotExists) ? ChangeType.CREATE : ChangeType.UPSERT) - .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr())) - .recordTemplate( - GenericRecordUtils.deserializeAspect( - ByteString.copyString(jsonAspect, StandardCharsets.UTF_8), - GenericRecordUtils.JSON, - aspectSpec)) - .build(aspectRetriever); - } - - private ChangeMCP toUpsertItem( - @Nonnull AspectRetriever aspectRetriever, - @Nonnull Urn urn, - @Nonnull AspectSpec aspectSpec, - @Nullable RecordTemplate currentValue, - @Nonnull GenericPatchTemplate genericPatchTemplate, - @Nonnull Actor actor) { - return ChangeItemImpl.fromPatch( - urn, - aspectSpec, - currentValue, - genericPatchTemplate, - AuditStampUtils.createAuditStamp(actor.toUrnStr()), - aspectRetriever); + return GenericEntityScrollResultV2.builder() + .results(toRecordTemplates(opContext, searchEntities, aspectNames, withSystemMetadata)) + .scrollId(scrollId) + .build(); } - private AspectsBatch toMCPBatch( + @Override + protected AspectsBatch toMCPBatch( @Nonnull OperationContext opContext, String entityArrayList, Actor actor) - throws JsonProcessingException, URISyntaxException { + throws JsonProcessingException, InvalidUrnException { JsonNode entities = objectMapper.readTree(entityArrayList); List items = new LinkedList<>(); @@ -707,8 +139,14 @@ private AspectsBatch toMCPBatch( Iterator entityItr = entities.iterator(); while (entityItr.hasNext()) { JsonNode entity = entityItr.next(); - Urn entityUrn = UrnUtils.getUrn(entity.get("urn").asText()); + if (!entity.has("urn")) { + throw new IllegalArgumentException("Missing `urn` field"); + } + Urn entityUrn = validatedUrn(entity.get("urn").asText()); + if (!entity.has("aspects")) { + throw new IllegalArgumentException("Missing `aspects` field"); + } Iterator> aspectItr = entity.get("aspects").fields(); while (aspectItr.hasNext()) { Map.Entry aspect = aspectItr.next(); @@ -747,9 +185,60 @@ private AspectsBatch toMCPBatch( .build(); } - public List toEntityListResponse( + @Override + protected List buildEntityVersionedAspectList( + @Nonnull OperationContext opContext, + LinkedHashMap> urnAspectVersions, + boolean withSystemMetadata) + throws URISyntaxException { + Map> aspects = + entityService.getEnvelopedVersionedAspects( + opContext, resolveAspectNames(urnAspectVersions, 0L), true); + + return urnAspectVersions.keySet().stream() + .map( + u -> + GenericEntityV2.builder() + .urn(u.toString()) + .build( + objectMapper, + toAspectMap(u, aspects.getOrDefault(u, List.of()), withSystemMetadata))) + .collect(Collectors.toList()); + } + + @Override + protected GenericEntityV2 buildGenericEntity( + @Nonnull String aspectName, + @Nonnull UpdateAspectResult updateAspectResult, + boolean withSystemMetadata) { + return GenericEntityV2.builder() + .urn(updateAspectResult.getUrn().toString()) + .build( + objectMapper, + Map.of( + aspectName, + Pair.of( + updateAspectResult.getNewValue(), + withSystemMetadata ? updateAspectResult.getNewSystemMetadata() : null))); + } + + private List toRecordTemplates( + @Nonnull OperationContext opContext, + SearchEntityArray searchEntities, + Set aspectNames, + boolean withSystemMetadata) + throws URISyntaxException { + return buildEntityList( + opContext, + searchEntities.stream().map(SearchEntity::getEntity).collect(Collectors.toList()), + aspectNames, + withSystemMetadata); + } + + @Override + protected List buildEntityList( Set ingestResults, boolean withSystemMetadata) { - List responseList = new LinkedList<>(); + List responseList = new LinkedList<>(); Map> entityMap = ingestResults.stream().collect(Collectors.groupingBy(IngestResult::getUrn)); @@ -765,24 +254,10 @@ public List toEntityListResponse( withSystemMetadata ? ingest.getRequest().getSystemMetadata() : null))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); responseList.add( - GenericEntity.builder() + GenericEntityV2.builder() .urn(urnAspects.getKey().toString()) .build(objectMapper, aspectsMap)); } return responseList; } - - /** - * Case-insensitive fallback - * - * @return - */ - private static AspectSpec lookupAspectSpec(EntitySpec entitySpec, String aspectName) { - return entitySpec.getAspectSpec(aspectName) != null - ? entitySpec.getAspectSpec(aspectName) - : entitySpec.getAspectSpecs().stream() - .filter(aspec -> aspec.getName().toLowerCase().equals(aspectName)) - .findFirst() - .get(); - } } diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/PlatformEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/PlatformEntitiesController.java index 1d0de44c0d83a..87c72064ad7a7 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/PlatformEntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/PlatformEntitiesController.java @@ -17,6 +17,7 @@ import io.datahubproject.openapi.generated.MetadataChangeProposal; import io.datahubproject.openapi.util.MappingUtil; import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -58,6 +59,7 @@ public void initBinder(WebDataBinder binder) { @PostMapping(value = "/", produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity> postEntities( + HttpServletRequest request, @RequestBody @Nonnull List metadataChangeProposals, @RequestParam(required = false, name = "async") Boolean async) { log.info("INGEST PROPOSAL proposal: {}", metadataChangeProposals); @@ -69,6 +71,8 @@ public ResponseEntity> postEntities( systemOperationContext, RequestContext.builder() .buildOpenapi( + actorUrnStr, + request, "postEntities", metadataChangeProposals.stream() .map(MetadataChangeProposal::getEntityType) diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/RelationshipController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/RelationshipController.java index ac0b9dd8c03ef..3e46e10857fbd 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/RelationshipController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/RelationshipController.java @@ -18,8 +18,8 @@ import com.linkedin.metadata.query.filter.RelationshipFilter; import com.linkedin.metadata.search.utils.QueryUtils; import io.datahubproject.openapi.exception.UnauthorizedException; +import io.datahubproject.openapi.models.GenericScrollResult; import io.datahubproject.openapi.v2.models.GenericRelationship; -import io.datahubproject.openapi.v2.models.GenericScrollResult; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import java.util.Arrays; diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/TimelineController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/TimelineControllerV2.java similarity index 99% rename from metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/TimelineController.java rename to metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/TimelineControllerV2.java index f010ef8e154b8..29b7de6ae5e8f 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/TimelineController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/TimelineControllerV2.java @@ -36,7 +36,7 @@ name = "Timeline", description = "An API for retrieving historical updates to entities and their related documentation.") -public class TimelineController { +public class TimelineControllerV2 { private final TimelineService _timelineService; private final AuthorizerChain _authorizerChain; diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/TimeseriesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/TimeseriesController.java index 267122d71a57b..bb10719bacd3f 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/TimeseriesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/TimeseriesController.java @@ -19,9 +19,10 @@ import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.metadata.context.RequestContext; import io.datahubproject.openapi.exception.UnauthorizedException; -import io.datahubproject.openapi.v2.models.GenericScrollResult; +import io.datahubproject.openapi.models.GenericScrollResult; import io.datahubproject.openapi.v2.models.GenericTimeseriesAspect; import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; import java.net.URISyntaxException; import java.util.List; import java.util.stream.Collectors; @@ -56,6 +57,7 @@ public class TimeseriesController { @GetMapping(value = "/{entityName}/{aspectName}", produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity> getAspects( + HttpServletRequest request, @PathVariable("entityName") String entityName, @PathVariable("aspectName") String aspectName, @RequestParam(value = "count", defaultValue = "10") Integer count, @@ -76,7 +78,9 @@ public ResponseEntity> getAspects( OperationContext opContext = OperationContext.asSession( systemOperationContext, - RequestContext.builder().buildOpenapi("getAspects", entityName), + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), request, "getAspects", entityName), authorizationChain, authentication, true); diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java index df3f6445a855a..3a93eb304b8f8 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java @@ -41,7 +41,9 @@ public class OpenAPIV3Generator { private static final String NAME_PATH = "path"; private static final String NAME_SYSTEM_METADATA = "systemMetadata"; private static final String NAME_ASYNC = "async"; + private static final String NAME_VERSION = "version"; private static final String NAME_SCROLL_ID = "scrollId"; + private static final String NAME_INCLUDE_SOFT_DELETE = "includeSoftDelete"; private static final String PROPERTY_VALUE = "value"; private static final String PROPERTY_URN = "urn"; private static final String PROPERTY_PATCH = "patch"; @@ -56,6 +58,7 @@ public class OpenAPIV3Generator { private static final String ASPECT_RESPONSE_SUFFIX = "Aspect" + RESPONSE_SUFFIX; private static final String ENTITY_REQUEST_SUFFIX = "Entity" + REQUEST_SUFFIX; private static final String ENTITY_RESPONSE_SUFFIX = "Entity" + RESPONSE_SUFFIX; + private static final String NAME_SKIP_CACHE = "skipCache"; public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) { final Set aspectNames = entityRegistry.getAspectSpecs().keySet(); @@ -105,6 +108,9 @@ public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) { entityName + ENTITY_RESPONSE_SUFFIX, buildEntitySchema(e, aspectNames, true)); components.addSchemas( "Scroll" + entityName + ENTITY_RESPONSE_SUFFIX, buildEntityScrollSchema(e)); + components.addSchemas( + "BatchGet" + entityName + ENTITY_REQUEST_SUFFIX, + buildEntityBatchGetRequestSchema(e, aspectNames)); }); // Parameters entityRegistry.getEntitySpecs().values().stream() @@ -125,6 +131,9 @@ public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) { paths.addPathItem( String.format("/v3/entity/%s", e.getName().toLowerCase()), buildListEntityPath(e)); + paths.addPathItem( + String.format("/v3/entity/%s/batchGet", e.getName().toLowerCase()), + buildBatchGetEntityPath(e)); paths.addPathItem( String.format("/v3/entity/%s/{urn}", e.getName().toLowerCase()), buildSingleEntityPath(e)); @@ -208,7 +217,12 @@ private static PathItem buildSingleEntityPath(final EntitySpec entity) { .in(NAME_PATH) .name("urn") .description("The entity's unique URN id.") - .schema(new Schema().type(TYPE_STRING)))) + .schema(new Schema().type(TYPE_STRING)), + new Parameter() + .in(NAME_QUERY) + .name(NAME_INCLUDE_SOFT_DELETE) + .description("If enabled, soft deleted items will exist.") + .schema(new Schema().type(TYPE_BOOLEAN)._default(false)))) .tags(List.of(entity.getName() + " Entity")) .responses( new ApiResponses() @@ -243,9 +257,19 @@ private static PathItem buildListEntityPath(final EntitySpec entity) { List.of( new Parameter() .in(NAME_QUERY) - .name("systemMetadata") + .name(NAME_SYSTEM_METADATA) .description("Include systemMetadata with response.") .schema(new Schema().type(TYPE_BOOLEAN)._default(false)), + new Parameter() + .in(NAME_QUERY) + .name(NAME_INCLUDE_SOFT_DELETE) + .description("Include soft-deleted aspects with response.") + .schema(new Schema().type(TYPE_BOOLEAN)._default(false)), + new Parameter() + .in(NAME_QUERY) + .name(NAME_SKIP_CACHE) + .description("Skip cache when listing entities.") + .schema(new Schema().type(TYPE_BOOLEAN)._default(false)), new Parameter() .$ref( String.format( @@ -274,6 +298,7 @@ private static PathItem buildListEntityPath(final EntitySpec entity) { .summary(String.format("Scroll/List %s.", upperFirst)) .parameters(parameters) .tags(List.of(entity.getName() + " Entity")) + .description("Scroll indexed entities. Will not include soft deleted entities.") .responses(new ApiResponses().addApiResponse("200", successApiResponse))); // Post Operation @@ -342,6 +367,62 @@ private static PathItem buildListEntityPath(final EntitySpec entity) { return result; } + private static PathItem buildBatchGetEntityPath(final EntitySpec entity) { + final String upperFirst = toUpperFirst(entity.getName()); + final PathItem result = new PathItem(); + // Post Operation + final Content requestBatch = + new Content() + .addMediaType( + "application/json", + new MediaType() + .schema( + new Schema() + .type(TYPE_ARRAY) + .items( + new Schema() + .$ref( + String.format( + "#/components/schemas/%s%s", + "BatchGet" + upperFirst, ENTITY_REQUEST_SUFFIX))))); + final ApiResponse apiBatchGetResponse = + new ApiResponse() + .description("Get a batch of " + entity.getName() + " entities.") + .content( + new Content() + .addMediaType( + "application/json", + new MediaType() + .schema( + new Schema() + .type(TYPE_ARRAY) + .items( + new Schema<>() + .$ref( + String.format( + "#/components/schemas/%s%s", + upperFirst, ENTITY_RESPONSE_SUFFIX)))))); + result.setPost( + new Operation() + .summary("Batch Get " + upperFirst + " entities.") + .tags(List.of(entity.getName() + " Entity")) + .parameters( + List.of( + new Parameter() + .in(NAME_QUERY) + .name("systemMetadata") + .description("Include systemMetadata with response.") + .schema(new Schema().type(TYPE_BOOLEAN)._default(false)))) + .requestBody( + new RequestBody() + .description("Batch Get " + entity.getName() + " entities.") + .required(true) + .content(requestBatch)) + .responses(new ApiResponses().addApiResponse("200", apiBatchGetResponse))); + + return result; + } + private static void addExtraParameters(final Components components) { components.addParameters( "ScrollId" + MODEL_VERSION, @@ -437,27 +518,31 @@ private static void addAspectSchemas(final Components components, final AspectSp final String newDefinition = definition.replaceAll("definitions", "components/schemas"); Schema s = Json.mapper().readValue(newDefinition, Schema.class); - Set requiredNames = - Optional.ofNullable(s.getRequired()) - .map(names -> Set.copyOf(names)) - .orElse(new HashSet()); - Map properties = - Optional.ofNullable(s.getProperties()).orElse(new HashMap<>()); - properties.forEach( - (name, schema) -> { - String $ref = schema.get$ref(); - boolean isNameRequired = requiredNames.contains(name); - if ($ref != null && !isNameRequired) { - // A non-required $ref property must be wrapped in a { allOf: [ $ref ] } - // object to allow the - // property to be marked as nullable - schema.setType(TYPE_OBJECT); - schema.set$ref(null); - schema.setAllOf(List.of(new Schema().$ref($ref))); - } - schema.setNullable(!isNameRequired); - }); - + // Set enums to "string". + if (s.getEnum() != null && !s.getEnum().isEmpty()) { + s.setType("string"); + } else { + Set requiredNames = + Optional.ofNullable(s.getRequired()) + .map(names -> Set.copyOf(names)) + .orElse(new HashSet()); + Map properties = + Optional.ofNullable(s.getProperties()).orElse(new HashMap<>()); + properties.forEach( + (name, schema) -> { + String $ref = schema.get$ref(); + boolean isNameRequired = requiredNames.contains(name); + if ($ref != null && !isNameRequired) { + // A non-required $ref property must be wrapped in a { allOf: [ $ref ] } + // object to allow the + // property to be marked as nullable + schema.setType(TYPE_OBJECT); + schema.set$ref(null); + schema.setAllOf(List.of(new Schema().$ref($ref))); + } + schema.setNullable(!isNameRequired); + }); + } components.addSchemas(n, s); } catch (Exception e) { throw new RuntimeException(e); @@ -486,7 +571,30 @@ private static Schema buildAspectRefResponseSchema(final String aspectName) { } private static Schema buildAspectRefRequestSchema(final String aspectName) { - return new Schema<>().$ref(PATH_DEFINITIONS + aspectName); + final Schema result = + new Schema<>() + .type(TYPE_OBJECT) + .description(ASPECT_DESCRIPTION) + .required(List.of(PROPERTY_VALUE)) + .addProperty(PROPERTY_VALUE, new Schema<>().$ref(PATH_DEFINITIONS + aspectName)); + result.addProperty( + "systemMetadata", + new Schema<>() + .type(TYPE_OBJECT) + .allOf(List.of(new Schema().$ref(PATH_DEFINITIONS + "SystemMetadata"))) + .description("System metadata for the aspect.") + .nullable(true)); + + Schema stringTypeSchema = new Schema<>(); + stringTypeSchema.setType(TYPE_STRING); + result.addProperty( + "headers", + new Schema<>() + .type(TYPE_OBJECT) + .additionalProperties(stringTypeSchema) + .description("System headers for the operation.") + .nullable(true)); + return result; } private static Schema buildEntitySchema( @@ -534,15 +642,57 @@ private static Schema buildEntityScrollSchema(final EntitySpec entity) { toUpperFirst(entity.getName()), ENTITY_RESPONSE_SUFFIX)))); } + private static Schema buildEntityBatchGetRequestSchema( + final EntitySpec entity, Set aspectNames) { + + final Schema stringTypeSchema = new Schema<>(); + stringTypeSchema.setType(TYPE_STRING); + final Map headers = + Map.of( + "headers", + new Schema<>() + .type(TYPE_OBJECT) + .additionalProperties(stringTypeSchema) + .description("System headers for the operation.") + .nullable(true)); + + final Map properties = + entity.getAspectSpecMap().entrySet().stream() + .filter(a -> aspectNames.contains(a.getValue().getName())) + .collect( + Collectors.toMap( + Map.Entry::getKey, a -> new Schema().type(TYPE_OBJECT).properties(headers))); + properties.put( + PROPERTY_URN, + new Schema<>().type(TYPE_STRING).description("Unique id for " + entity.getName())); + + properties.put(entity.getKeyAspectName(), new Schema().type(TYPE_OBJECT).properties(headers)); + + return new Schema<>() + .type(TYPE_OBJECT) + .description(toUpperFirst(entity.getName()) + " object.") + .required(List.of(PROPERTY_URN)) + .properties(properties); + } + private static Schema buildAspectRef(final String aspect, final boolean withSystemMetadata) { + // A non-required $ref property must be wrapped in a { allOf: [ $ref ] } + // object to allow the + // property to be marked as nullable final Schema result = new Schema<>(); + + result.setType(TYPE_OBJECT); + result.set$ref(null); + result.setNullable(true); + final String internalRef; if (withSystemMetadata) { - result.set$ref( - String.format(FORMAT_PATH_DEFINITIONS, toUpperFirst(aspect), ASPECT_RESPONSE_SUFFIX)); + internalRef = + String.format(FORMAT_PATH_DEFINITIONS, toUpperFirst(aspect), ASPECT_RESPONSE_SUFFIX); } else { - result.set$ref( - String.format(FORMAT_PATH_DEFINITIONS, toUpperFirst(aspect), ASPECT_REQUEST_SUFFIX)); + internalRef = + String.format(FORMAT_PATH_DEFINITIONS, toUpperFirst(aspect), ASPECT_REQUEST_SUFFIX); } + result.setAllOf(List.of(new Schema().$ref(internalRef))); return result; } @@ -585,6 +735,12 @@ private static PathItem buildSingleEntityAspectPath( .name(NAME_SYSTEM_METADATA) .description("Include systemMetadata with response.") .schema(new Schema().type(TYPE_BOOLEAN)._default(false)); + final Parameter versionParameter = + new Parameter() + .in(NAME_QUERY) + .name(NAME_VERSION) + .description("Return a specific aspect version.") + .schema(new Schema().type(TYPE_INTEGER)._default(0).minimum(BigDecimal.ZERO)); final ApiResponse successApiResponse = new ApiResponse() .description("Success") @@ -603,7 +759,7 @@ private static PathItem buildSingleEntityAspectPath( new Operation() .summary(String.format("Get %s for %s.", aspect, entity.getName())) .tags(tags) - .parameters(List.of(getParameter)) + .parameters(List.of(getParameter, versionParameter)) .responses(new ApiResponses().addApiResponse("200", successApiResponse)); // Head Operation final ApiResponse successHeadResponse = @@ -618,6 +774,13 @@ private static PathItem buildSingleEntityAspectPath( new Operation() .summary(String.format("%s on %s existence.", aspect, upperFirstEntity)) .tags(tags) + .parameters( + List.of( + new Parameter() + .in(NAME_QUERY) + .name(NAME_INCLUDE_SOFT_DELETE) + .description("If enabled, soft deleted items will exist.") + .schema(new Schema().type(TYPE_BOOLEAN)._default(false)))) .responses( new ApiResponses() .addApiResponse("200", successHeadResponse) diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java index 689efbf8bc6ec..d6feb6cc460c9 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java @@ -1,12 +1,325 @@ package io.datahubproject.openapi.v3.controller; +import static com.linkedin.metadata.aspect.validation.ConditionalWriteValidator.HTTP_HEADER_IF_VERSION_MATCH; +import static com.linkedin.metadata.authorization.ApiOperation.READ; + +import com.datahub.authentication.Actor; +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; +import com.datahub.authorization.AuthUtil; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.ByteString; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.metadata.aspect.batch.AspectsBatch; +import com.linkedin.metadata.aspect.batch.BatchItem; +import com.linkedin.metadata.entity.EntityApiUtils; +import com.linkedin.metadata.entity.IngestResult; +import com.linkedin.metadata.entity.UpdateAspectResult; +import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; +import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.utils.AuditStampUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.SystemMetadata; +import com.linkedin.util.Pair; +import io.datahubproject.metadata.context.OperationContext; +import io.datahubproject.metadata.context.RequestContext; +import io.datahubproject.openapi.controller.GenericEntitiesController; +import io.datahubproject.openapi.exception.InvalidUrnException; +import io.datahubproject.openapi.exception.UnauthorizedException; +import io.datahubproject.openapi.v3.models.GenericAspectV3; +import io.datahubproject.openapi.v3.models.GenericEntityScrollResultV3; +import io.datahubproject.openapi.v3.models.GenericEntityV3; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; @RestController("EntityControllerV3") @RequiredArgsConstructor @RequestMapping("/v3/entity") @Slf4j -public class EntityController extends io.datahubproject.openapi.v2.controller.EntityController {} +public class EntityController + extends GenericEntitiesController< + GenericAspectV3, GenericEntityV3, GenericEntityScrollResultV3> { + + @Tag(name = "Generic Entities") + @PostMapping(value = "/{entityName}/batchGet", produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Get a batch of entities") + public ResponseEntity> getEntityBatch( + HttpServletRequest httpServletRequest, + @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") + Boolean withSystemMetadata, + @RequestBody @Nonnull String jsonEntityList) + throws URISyntaxException, JsonProcessingException { + + LinkedHashMap> requestMap = toEntityVersionRequest(jsonEntityList); + + Authentication authentication = AuthenticationContext.getAuthentication(); + OperationContext opContext = + OperationContext.asSession( + systemOperationContext, + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), + httpServletRequest, + "getEntityBatch", + requestMap.keySet().stream() + .map(Urn::getEntityType) + .collect(Collectors.toSet())), + authorizationChain, + authentication, + true); + + if (!AuthUtil.isAPIAuthorizedEntityUrns( + authentication, authorizationChain, READ, requestMap.keySet())) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + " is unauthorized to " + READ + " entities."); + } + + return ResponseEntity.of( + Optional.of(buildEntityVersionedAspectList(opContext, requestMap, withSystemMetadata))); + } + + @Override + public GenericEntityScrollResultV3 buildScrollResult( + @Nonnull OperationContext opContext, + SearchEntityArray searchEntities, + Set aspectNames, + boolean withSystemMetadata, + @Nullable String scrollId) + throws URISyntaxException { + return GenericEntityScrollResultV3.builder() + .entities(toRecordTemplates(opContext, searchEntities, aspectNames, withSystemMetadata)) + .scrollId(scrollId) + .build(); + } + + @Override + protected List buildEntityVersionedAspectList( + @Nonnull OperationContext opContext, + LinkedHashMap> urnAspectVersions, + boolean withSystemMetadata) + throws URISyntaxException { + if (urnAspectVersions.isEmpty()) { + return List.of(); + } else { + Map> aspects = + entityService.getEnvelopedVersionedAspects( + opContext, resolveAspectNames(urnAspectVersions, 0L), false); + + return urnAspectVersions.keySet().stream() + .filter(urn -> aspects.containsKey(urn) && !aspects.get(urn).isEmpty()) + .map( + u -> + GenericEntityV3.builder() + .build(objectMapper, u, toAspectMap(u, aspects.get(u), withSystemMetadata))) + .collect(Collectors.toList()); + } + } + + @Override + protected List buildEntityList( + Set ingestResults, boolean withSystemMetadata) { + List responseList = new LinkedList<>(); + + Map> entityMap = + ingestResults.stream().collect(Collectors.groupingBy(IngestResult::getUrn)); + for (Map.Entry> urnAspects : entityMap.entrySet()) { + Map> aspectsMap = + urnAspects.getValue().stream() + .map( + ingest -> + Map.entry( + ingest.getRequest().getAspectName(), + Pair.of( + ingest.getRequest().getRecordTemplate(), + withSystemMetadata ? ingest.getRequest().getSystemMetadata() : null))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + responseList.add( + GenericEntityV3.builder().build(objectMapper, urnAspects.getKey(), aspectsMap)); + } + return responseList; + } + + @Override + protected GenericEntityV3 buildGenericEntity( + @Nonnull String aspectName, + @Nonnull UpdateAspectResult updateAspectResult, + boolean withSystemMetadata) { + return GenericEntityV3.builder() + .build( + objectMapper, + updateAspectResult.getUrn(), + Map.of( + aspectName, + Pair.of( + updateAspectResult.getNewValue(), + withSystemMetadata ? updateAspectResult.getNewSystemMetadata() : null))); + } + + private List toRecordTemplates( + @Nonnull OperationContext opContext, + SearchEntityArray searchEntities, + Set aspectNames, + boolean withSystemMetadata) + throws URISyntaxException { + return buildEntityList( + opContext, + searchEntities.stream().map(SearchEntity::getEntity).collect(Collectors.toList()), + aspectNames, + withSystemMetadata); + } + + private LinkedHashMap> toEntityVersionRequest( + @Nonnull String entityArrayList) throws JsonProcessingException, InvalidUrnException { + JsonNode entities = objectMapper.readTree(entityArrayList); + + LinkedHashMap> items = new LinkedHashMap<>(); + if (entities.isArray()) { + Iterator entityItr = entities.iterator(); + while (entityItr.hasNext()) { + JsonNode entity = entityItr.next(); + if (!entity.has("urn")) { + throw new IllegalArgumentException("Missing `urn` field"); + } + Urn entityUrn = validatedUrn(entity.get("urn").asText()); + items.putIfAbsent(entityUrn, new HashMap<>()); + + Iterator> aspectItr = entity.fields(); + while (aspectItr.hasNext()) { + Map.Entry aspect = aspectItr.next(); + + if ("urn".equals(aspect.getKey())) { + continue; + } + + AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey()); + + if (aspectSpec != null) { + + Map headers = null; + if (aspect.getValue().has("headers")) { + headers = + objectMapper.convertValue( + aspect.getValue().get("headers"), new TypeReference<>() {}); + items + .get(entityUrn) + .put( + aspectSpec.getName(), + Long.parseLong(headers.getOrDefault(HTTP_HEADER_IF_VERSION_MATCH, "0"))); + } else { + items.get(entityUrn).put(aspectSpec.getName(), 0L); + } + } + } + + // handle no aspects specified, default latest version + if (items.get(entityUrn).isEmpty()) { + for (AspectSpec aspectSpec : + entityRegistry.getEntitySpec(entityUrn.getEntityType()).getAspectSpecs()) { + items.get(entityUrn).put(aspectSpec.getName(), 0L); + } + } + } + } + + return items; + } + + @Override + protected AspectsBatch toMCPBatch( + @Nonnull OperationContext opContext, String entityArrayList, Actor actor) + throws JsonProcessingException, InvalidUrnException { + JsonNode entities = objectMapper.readTree(entityArrayList); + + List items = new LinkedList<>(); + if (entities.isArray()) { + Iterator entityItr = entities.iterator(); + while (entityItr.hasNext()) { + JsonNode entity = entityItr.next(); + if (!entity.has("urn")) { + throw new IllegalArgumentException("Missing `urn` field"); + } + Urn entityUrn = validatedUrn(entity.get("urn").asText()); + + Iterator> aspectItr = entity.fields(); + while (aspectItr.hasNext()) { + Map.Entry aspect = aspectItr.next(); + + if ("urn".equals(aspect.getKey())) { + continue; + } + + AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey()); + + if (aspectSpec != null) { + + SystemMetadata systemMetadata = null; + if (aspect.getValue().has("systemMetadata")) { + systemMetadata = + EntityApiUtils.parseSystemMetadata( + objectMapper.writeValueAsString(aspect.getValue().get("systemMetadata"))); + ((ObjectNode) aspect.getValue()).remove("systemMetadata"); + } + Map headers = null; + if (aspect.getValue().has("headers")) { + headers = + objectMapper.convertValue( + aspect.getValue().get("headers"), new TypeReference<>() {}); + } + + ChangeItemImpl.ChangeItemImplBuilder builder = + ChangeItemImpl.builder() + .urn(entityUrn) + .aspectName(aspectSpec.getName()) + .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr())) + .systemMetadata(systemMetadata) + .headers(headers) + .recordTemplate( + GenericRecordUtils.deserializeAspect( + ByteString.copyString( + objectMapper.writeValueAsString(aspect.getValue().get("value")), + StandardCharsets.UTF_8), + GenericRecordUtils.JSON, + aspectSpec)); + + items.add(builder.build(opContext.getRetrieverContext().get().getAspectRetriever())); + } + } + } + } + return AspectsBatchImpl.builder() + .items(items) + .retrieverContext(opContext.getRetrieverContext().get()) + .build(); + } +} diff --git a/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java b/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java index 8e70bd507999f..3e352403c88bc 100644 --- a/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java +++ b/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java @@ -214,7 +214,7 @@ public void testIngestDataset() { .build(); datasetAspects.add(glossaryTerms); - _entitiesController.postEntities(datasetAspects, false, false, false); + _entitiesController.postEntities(null, datasetAspects, false, false, false); } // @Test diff --git a/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/OpenAPIV3GeneratorTest.java b/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/OpenAPIV3GeneratorTest.java index 0ce62f5cb10f6..10b75fd7faed3 100644 --- a/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/OpenAPIV3GeneratorTest.java +++ b/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/OpenAPIV3GeneratorTest.java @@ -83,5 +83,10 @@ public void testOpenApiSpecBuilder() throws Exception { List.of(new Schema().$ref("#/components/schemas/SystemMetadata")), systemMetadata.getAllOf()); assertTrue(systemMetadata.getNullable()); + + // Assert enum property is string. + Schema fabricType = openAPI.getComponents().getSchemas().get("FabricType"); + assertEquals("string", fabricType.getType()); + assertFalse(fabricType.getEnum().isEmpty()); } } diff --git a/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/controller/EntityControllerTest.java b/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/controller/EntityControllerTest.java new file mode 100644 index 0000000000000..0855ad6c2e4ff --- /dev/null +++ b/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/controller/EntityControllerTest.java @@ -0,0 +1,205 @@ +package io.datahubproject.openapi.v3.controller; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.testng.Assert.assertNotNull; + +import com.datahub.authentication.Actor; +import com.datahub.authentication.ActorType; +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; +import com.datahub.authorization.AuthorizationResult; +import com.datahub.authorization.AuthorizerChain; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityServiceImpl; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.query.filter.SortOrder; +import com.linkedin.metadata.search.ScrollResult; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.utils.SearchUtil; +import io.datahubproject.metadata.context.OperationContext; +import io.datahubproject.openapi.config.SpringWebConfig; +import io.datahubproject.test.metadata.context.TestOperationContexts; +import java.util.List; +import java.util.Map; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureWebMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.http.MediaType; +import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; +import org.testng.annotations.Test; + +@SpringBootTest(classes = {SpringWebConfig.class}) +@ComponentScan(basePackages = {"io.datahubproject.openapi.v3.controller"}) +@Import({SpringWebConfig.class, EntityControllerTest.EntityControllerTestConfig.class}) +@AutoConfigureWebMvc +@AutoConfigureMockMvc +public class EntityControllerTest extends AbstractTestNGSpringContextTests { + @Autowired private EntityController entityController; + @Autowired private MockMvc mockMvc; + @Autowired private SearchService mockSearchService; + @Autowired private EntityService mockEntityService; + + @Test + public void initTest() { + assertNotNull(entityController); + } + + @Test + public void testSearchOrderPreserved() throws Exception { + List TEST_URNS = + List.of( + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:testPlatform,1,PROD)"), + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:testPlatform,2,PROD)"), + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:testPlatform,3,PROD)")); + + // Mock scroll ascending/descending results + ScrollResult expectedResultAscending = + new ScrollResult() + .setEntities( + new SearchEntityArray( + List.of( + new SearchEntity().setEntity(TEST_URNS.get(0)), + new SearchEntity().setEntity(TEST_URNS.get(1)), + new SearchEntity().setEntity(TEST_URNS.get(2))))); + when(mockSearchService.scrollAcrossEntities( + any(OperationContext.class), + eq(List.of("dataset")), + anyString(), + nullable(Filter.class), + eq(SearchUtil.sortBy("urn", SortOrder.valueOf("ASCENDING"))), + nullable(String.class), + nullable(String.class), + anyInt())) + .thenReturn(expectedResultAscending); + ScrollResult expectedResultDescending = + new ScrollResult() + .setEntities( + new SearchEntityArray( + List.of( + new SearchEntity().setEntity(TEST_URNS.get(2)), + new SearchEntity().setEntity(TEST_URNS.get(1)), + new SearchEntity().setEntity(TEST_URNS.get(0))))); + when(mockSearchService.scrollAcrossEntities( + any(OperationContext.class), + eq(List.of("dataset")), + anyString(), + nullable(Filter.class), + eq(SearchUtil.sortBy("urn", SortOrder.valueOf("DESCENDING"))), + nullable(String.class), + nullable(String.class), + anyInt())) + .thenReturn(expectedResultDescending); + // Mock entity aspect + when(mockEntityService.getEnvelopedVersionedAspects( + any(OperationContext.class), anyMap(), eq(false))) + .thenReturn( + Map.of( + TEST_URNS.get(0), + List.of( + new EnvelopedAspect() + .setName("status") + .setValue(new Aspect(new Status().data()))), + TEST_URNS.get(1), + List.of( + new EnvelopedAspect() + .setName("status") + .setValue(new Aspect(new Status().data()))), + TEST_URNS.get(2), + List.of( + new EnvelopedAspect() + .setName("status") + .setValue(new Aspect(new Status().data()))))); + + // test ASCENDING + mockMvc + .perform( + MockMvcRequestBuilders.get("/v3/entity/dataset") + .param("sortOrder", "ASCENDING") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().is2xxSuccessful()) + .andExpect( + MockMvcResultMatchers.jsonPath("$.entities[0].urn").value(TEST_URNS.get(0).toString())) + .andExpect( + MockMvcResultMatchers.jsonPath("$.entities[1].urn").value(TEST_URNS.get(1).toString())) + .andExpect( + MockMvcResultMatchers.jsonPath("$.entities[2].urn").value(TEST_URNS.get(2).toString())); + + // test DESCENDING + mockMvc + .perform( + MockMvcRequestBuilders.get("/v3/entity/dataset") + .accept(MediaType.APPLICATION_JSON) + .param("sortOrder", "DESCENDING")) + .andExpect(status().is2xxSuccessful()) + .andExpect( + MockMvcResultMatchers.jsonPath("$.entities[0].urn").value(TEST_URNS.get(2).toString())) + .andExpect( + MockMvcResultMatchers.jsonPath("$.entities[1].urn").value(TEST_URNS.get(1).toString())) + .andExpect( + MockMvcResultMatchers.jsonPath("$.entities[2].urn").value(TEST_URNS.get(0).toString())); + } + + @TestConfiguration + public static class EntityControllerTestConfig { + @MockBean public EntityServiceImpl entityService; + @MockBean public SearchService searchService; + + @Bean + public ObjectMapper objectMapper() { + return new ObjectMapper(); + } + + @Bean(name = "systemOperationContext") + public OperationContext systemOperationContext() { + return TestOperationContexts.systemContextNoSearchAuthorization(); + } + + @Bean("entityRegistry") + @Primary + public EntityRegistry entityRegistry( + @Qualifier("systemOperationContext") final OperationContext testOperationContext) { + return testOperationContext.getEntityRegistry(); + } + + @Bean + public AuthorizerChain authorizerChain() { + AuthorizerChain authorizerChain = mock(AuthorizerChain.class); + + Authentication authentication = mock(Authentication.class); + when(authentication.getActor()).thenReturn(new Actor(ActorType.USER, "datahub")); + when(authorizerChain.authorize(any())) + .thenReturn(new AuthorizationResult(null, AuthorizationResult.Type.ALLOW, "")); + AuthenticationContext.setAuthentication(authentication); + + return authorizerChain; + } + } +} diff --git a/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java b/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java index 998ebe6a80096..b70b643b10f32 100644 --- a/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java +++ b/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java @@ -151,7 +151,8 @@ public Map> getLatestEnvelopedAspects( public Map> getVersionedEnvelopedAspects( @Nonnull OperationContext opContext, @Nonnull Set versionedUrns, - @Nonnull Set aspectNames) + @Nonnull Set aspectNames, + boolean alwaysIncludeKeyAspect) throws URISyntaxException { return null; } diff --git a/metadata-service/plugin/build.gradle b/metadata-service/plugin/build.gradle index f519eba4921d2..501589c24d60a 100644 --- a/metadata-service/plugin/build.gradle +++ b/metadata-service/plugin/build.gradle @@ -13,14 +13,18 @@ dependencies { implementation externalDependency.jacksonDataFormatYaml implementation externalDependency.jacksonJDK8 implementation externalDependency.jacksonDataPropertyFormat - implementation externalDependency.logbackClassic; + implementation externalDependency.logbackClassic implementation externalDependency.slf4jApi compileOnly externalDependency.lombok + annotationProcessor externalDependency.lombok + testImplementation project(':test-models') + testImplementation project(path: ':test-models', configuration: 'testDataTemplate') testImplementation externalDependency.mockito testImplementation externalDependency.testng - annotationProcessor externalDependency.lombok + testCompileOnly externalDependency.lombok + testAnnotationProcessor externalDependency.lombok } test { diff --git a/metadata-service/plugin/src/main/java/com/datahub/plugins/metadata/aspect/SpringPluginFactory.java b/metadata-service/plugin/src/main/java/com/datahub/plugins/metadata/aspect/SpringPluginFactory.java index 8a080c8d9076e..043b0016abaaa 100644 --- a/metadata-service/plugin/src/main/java/com/datahub/plugins/metadata/aspect/SpringPluginFactory.java +++ b/metadata-service/plugin/src/main/java/com/datahub/plugins/metadata/aspect/SpringPluginFactory.java @@ -14,6 +14,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.BeanFactoryUtils; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.AnnotationConfigApplicationContext; @@ -28,34 +29,39 @@ public SpringPluginFactory( @Nonnull List classLoaders) { super(pluginConfiguration, classLoaders); - String[] packageScan = - extractPackageScan( - Optional.ofNullable(pluginConfiguration) - .map(PluginConfiguration::streamAll) - .orElse(Stream.of())) - .toArray(String[]::new); - - if (springApplicationContext != null || packageScan.length == 0) { - this.springApplicationContext = springApplicationContext; - } else { - AnnotationConfigApplicationContext rootContext = null; - - for (ClassLoader classLoader : classLoaders) { - AnnotationConfigApplicationContext applicationContext = - new AnnotationConfigApplicationContext(); - applicationContext.setId("custom-plugin"); - if (rootContext != null) { - applicationContext.setParent(rootContext); + try { + String[] packageScan = + extractPackageScan( + Optional.ofNullable(pluginConfiguration) + .map(PluginConfiguration::streamAll) + .orElse(Stream.of())) + .toArray(String[]::new); + + if (springApplicationContext != null || packageScan.length == 0) { + this.springApplicationContext = springApplicationContext; + } else { + AnnotationConfigApplicationContext rootContext = null; + + for (ClassLoader classLoader : classLoaders) { + AnnotationConfigApplicationContext applicationContext = + new AnnotationConfigApplicationContext(); + applicationContext.setId("custom-plugin"); + if (rootContext != null) { + applicationContext.setParent(rootContext); + } + applicationContext.setClassLoader(classLoader); + applicationContext.scan(packageScan); + rootContext = applicationContext; } - applicationContext.setClassLoader(classLoader); - applicationContext.scan(packageScan); - rootContext = applicationContext; + rootContext.refresh(); + this.springApplicationContext = rootContext; } - rootContext.refresh(); - this.springApplicationContext = rootContext; - } - loadPlugins(); + loadPlugins(); + } catch (Exception e) { + log.error("Error loading Spring Plugins!", e); + throw e; + } } private static Stream extractPackageScan(Stream configStream) { @@ -101,16 +107,29 @@ protected List build( try { Class clazz = classLoader.loadClass(config.getClassName()); - final T plugin; + final List plugins; if (config.getSpring().getName() == null) { - plugin = (T) springApplicationContext.getBean(clazz); + plugins = + BeanFactoryUtils.beansOfTypeIncludingAncestors(springApplicationContext, clazz) + .values() + .stream() + .map(plugin -> (T) plugin) + .collect(Collectors.toList()); } else { - plugin = (T) springApplicationContext.getBean(config.getSpring().getName(), clazz); + plugins = + List.of((T) springApplicationContext.getBean(config.getSpring().getName(), clazz)); } - if (plugin.enabled()) { - result.add((T) plugin.setConfig(config)); - } + plugins.stream() + .filter(plugin -> plugin.enabled()) + .forEach( + plugin -> { + if (plugin.getConfig() != null) { + result.add(plugin); + } else { + result.add((T) plugin.setConfig(config)); + } + }); loaded = true; break; diff --git a/metadata-service/plugin/src/test/java/com/datahub/plugins/metadata/aspect/SpringPluginFactoryTest.java b/metadata-service/plugin/src/test/java/com/datahub/plugins/metadata/aspect/SpringPluginFactoryTest.java new file mode 100644 index 0000000000000..d321697304afc --- /dev/null +++ b/metadata-service/plugin/src/test/java/com/datahub/plugins/metadata/aspect/SpringPluginFactoryTest.java @@ -0,0 +1,188 @@ +package com.datahub.plugins.metadata.aspect; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + +import com.datahub.test.TestEntityProfile; +import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.RetrieverContext; +import com.linkedin.metadata.aspect.batch.BatchItem; +import com.linkedin.metadata.aspect.batch.ChangeMCP; +import com.linkedin.metadata.aspect.batch.MCLItem; +import com.linkedin.metadata.aspect.batch.MCPItem; +import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; +import com.linkedin.metadata.aspect.plugins.hooks.MCLSideEffect; +import com.linkedin.metadata.aspect.plugins.hooks.MCPSideEffect; +import com.linkedin.metadata.aspect.plugins.hooks.MutationHook; +import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator; +import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.EventSpec; +import com.linkedin.metadata.models.registry.ConfigEntityRegistry; +import com.linkedin.metadata.models.registry.EntityRegistryException; +import com.linkedin.metadata.models.registry.MergedEntityRegistry; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.springframework.context.annotation.Configuration; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +@Configuration +public class SpringPluginFactoryTest { + + public static String REGISTRY_FILE_1 = "test-entity-registry-plugins-1.yml"; + public static String REGISTRY_SPRING_FILE_1 = "test-entity-registry-spring-plugins-1.yml"; + + @BeforeTest + public void disableAssert() { + PathSpecBasedSchemaAnnotationVisitor.class + .getClassLoader() + .setClassAssertionStatus(PathSpecBasedSchemaAnnotationVisitor.class.getName(), false); + } + + @Test + public void testMergedEntityRegistryWithSpringPluginFactory() throws EntityRegistryException { + ConfigEntityRegistry configEntityRegistry1 = + new ConfigEntityRegistry( + TestEntityProfile.class.getClassLoader().getResourceAsStream(REGISTRY_FILE_1)); + ConfigEntityRegistry configEntityRegistry2 = + new ConfigEntityRegistry( + TestEntityProfile.class.getClassLoader().getResourceAsStream(REGISTRY_SPRING_FILE_1), + (config, classLoaders) -> + new SpringPluginFactory( + null, config, List.of(SpringPluginFactoryTest.class.getClassLoader()))); + + MergedEntityRegistry mergedEntityRegistry = new MergedEntityRegistry(configEntityRegistry1); + mergedEntityRegistry.apply(configEntityRegistry2); + + Map entitySpecs = mergedEntityRegistry.getEntitySpecs(); + Map eventSpecs = mergedEntityRegistry.getEventSpecs(); + assertEquals(entitySpecs.values().size(), 2); + assertEquals(eventSpecs.values().size(), 1); + + EntitySpec entitySpec = mergedEntityRegistry.getEntitySpec("dataset"); + assertEquals(entitySpec.getName(), "dataset"); + assertEquals(entitySpec.getKeyAspectSpec().getName(), "datasetKey"); + assertEquals(entitySpec.getAspectSpecs().size(), 4); + assertNotNull(entitySpec.getAspectSpec("datasetKey")); + assertNotNull(entitySpec.getAspectSpec("datasetProperties")); + assertNotNull(entitySpec.getAspectSpec("schemaMetadata")); + assertNotNull(entitySpec.getAspectSpec("status")); + + entitySpec = mergedEntityRegistry.getEntitySpec("chart"); + assertEquals(entitySpec.getName(), "chart"); + assertEquals(entitySpec.getKeyAspectSpec().getName(), "chartKey"); + assertEquals(entitySpec.getAspectSpecs().size(), 3); + assertNotNull(entitySpec.getAspectSpec("chartKey")); + assertNotNull(entitySpec.getAspectSpec("chartInfo")); + assertNotNull(entitySpec.getAspectSpec("status")); + + EventSpec eventSpec = mergedEntityRegistry.getEventSpec("testEvent"); + assertEquals(eventSpec.getName(), "testEvent"); + assertNotNull(eventSpec.getPegasusSchema()); + + assertEquals( + mergedEntityRegistry.getAllAspectPayloadValidators().stream() + .filter(validator -> validator.shouldApply(ChangeType.UPSERT, "chart", "status")) + .count(), + 2); + assertEquals( + mergedEntityRegistry.getAllAspectPayloadValidators().stream() + .filter(validator -> validator.shouldApply(ChangeType.DELETE, "chart", "status")) + .count(), + 1); + + assertEquals( + mergedEntityRegistry.getAllMCPSideEffects().stream() + .filter(validator -> validator.shouldApply(ChangeType.UPSERT, "dataset", "datasetKey")) + .count(), + 2); + assertEquals( + mergedEntityRegistry.getAllMCPSideEffects().stream() + .filter(validator -> validator.shouldApply(ChangeType.DELETE, "dataset", "datasetKey")) + .count(), + 1); + + assertEquals( + mergedEntityRegistry.getAllMutationHooks().stream() + .filter(validator -> validator.shouldApply(ChangeType.UPSERT, "*", "schemaMetadata")) + .count(), + 2); + assertEquals( + mergedEntityRegistry.getAllMutationHooks().stream() + .filter(validator -> validator.shouldApply(ChangeType.DELETE, "*", "schemaMetadata")) + .count(), + 1); + } + + /* + * Various test plugins to be injected with Spring + */ + @Getter + @Setter + @Accessors(chain = true) + public static class TestValidator extends AspectPayloadValidator { + + public AspectPluginConfig config; + + @Override + protected Stream validateProposedAspects( + @Nonnull Collection mcpItems, + @Nonnull RetrieverContext retrieverContext) { + return mcpItems.stream().map(i -> AspectValidationException.forItem(i, "test error")); + } + + @Override + protected Stream validatePreCommitAspects( + @Nonnull Collection changeMCPs, @Nonnull RetrieverContext retrieverContext) { + return Stream.empty(); + } + } + + @Getter + @Setter + @Accessors(chain = true) + public static class TestMutator extends MutationHook { + public AspectPluginConfig config; + } + + @Getter + @Setter + @Accessors(chain = true) + public static class TestMCPSideEffect extends MCPSideEffect { + + public AspectPluginConfig config; + + @Override + protected Stream applyMCPSideEffect( + Collection changeMCPS, @Nonnull RetrieverContext retrieverContext) { + return changeMCPS.stream(); + } + + @Override + protected Stream postMCPSideEffect( + Collection mclItems, @Nonnull RetrieverContext retrieverContext) { + return Stream.of(); + } + } + + @Getter + @Setter + @Accessors(chain = true) + public static class TestMCLSideEffect extends MCLSideEffect { + public AspectPluginConfig config; + + @Override + protected Stream applyMCLSideEffect( + @Nonnull Collection batchItems, @Nonnull RetrieverContext retrieverContext) { + return null; + } + } +} diff --git a/metadata-service/plugin/src/test/java/com/datahub/plugins/metadata/aspect/test/TestSpringPluginConfiguration.java b/metadata-service/plugin/src/test/java/com/datahub/plugins/metadata/aspect/test/TestSpringPluginConfiguration.java new file mode 100644 index 0000000000000..589ae614e2d2f --- /dev/null +++ b/metadata-service/plugin/src/test/java/com/datahub/plugins/metadata/aspect/test/TestSpringPluginConfiguration.java @@ -0,0 +1,109 @@ +package com.datahub.plugins.metadata.aspect.test; + +import com.datahub.plugins.metadata.aspect.SpringPluginFactoryTest; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; +import java.util.List; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** This is the Spring class used for the test */ +@Configuration +public class TestSpringPluginConfiguration { + @Bean + public SpringPluginFactoryTest.TestValidator springValidator1() { + SpringPluginFactoryTest.TestValidator testValidator = + new SpringPluginFactoryTest.TestValidator(); + testValidator.setConfig( + AspectPluginConfig.builder() + .supportedEntityAspectNames( + List.of( + AspectPluginConfig.EntityAspectName.builder() + .entityName("dataset") + .aspectName("status") + .build())) + .className(SpringPluginFactoryTest.TestValidator.class.getName()) + .enabled(true) + .supportedOperations(List.of(ChangeType.UPSERT.toString())) + .build()); + return testValidator; + } + + @Bean + public SpringPluginFactoryTest.TestValidator springValidator2() { + SpringPluginFactoryTest.TestValidator testValidator = + new SpringPluginFactoryTest.TestValidator(); + testValidator.setConfig( + AspectPluginConfig.builder() + .supportedEntityAspectNames( + List.of( + AspectPluginConfig.EntityAspectName.builder() + .entityName("*") + .aspectName("status") + .build())) + .className(SpringPluginFactoryTest.TestValidator.class.getName()) + .enabled(true) + .supportedOperations(List.of(ChangeType.DELETE.toString())) + .build()); + return testValidator; + } + + @Bean + public SpringPluginFactoryTest.TestMutator springMutator() { + SpringPluginFactoryTest.TestMutator testMutator = new SpringPluginFactoryTest.TestMutator(); + testMutator.setConfig( + AspectPluginConfig.builder() + .supportedEntityAspectNames( + List.of( + AspectPluginConfig.EntityAspectName.builder() + .entityName("*") + .aspectName("schemaMetadata") + .build())) + .className(SpringPluginFactoryTest.TestMutator.class.getName()) + .enabled(true) + .supportedOperations( + List.of(ChangeType.UPSERT.toString(), ChangeType.DELETE.toString())) + .build()); + return testMutator; + } + + @Bean + public SpringPluginFactoryTest.TestMCPSideEffect springMCPSideEffect() { + SpringPluginFactoryTest.TestMCPSideEffect testMCPSideEffect = + new SpringPluginFactoryTest.TestMCPSideEffect(); + testMCPSideEffect.setConfig( + AspectPluginConfig.builder() + .supportedEntityAspectNames( + List.of( + AspectPluginConfig.EntityAspectName.builder() + .entityName("dataset") + .aspectName("datasetKey") + .build())) + .className(SpringPluginFactoryTest.TestMCPSideEffect.class.getName()) + .enabled(true) + .supportedOperations( + List.of(ChangeType.UPSERT.toString(), ChangeType.DELETE.toString())) + .build()); + return testMCPSideEffect; + } + + @Bean + public SpringPluginFactoryTest.TestMCLSideEffect springMCLSideEffect() { + SpringPluginFactoryTest.TestMCLSideEffect testMCLSideEffect = + new SpringPluginFactoryTest.TestMCLSideEffect(); + testMCLSideEffect.setConfig( + AspectPluginConfig.builder() + .supportedEntityAspectNames( + List.of( + AspectPluginConfig.EntityAspectName.builder() + .entityName("chart") + .aspectName("ChartInfo") + .build())) + .className(SpringPluginFactoryTest.TestMCLSideEffect.class.getName()) + .enabled(true) + .supportedOperations( + List.of(ChangeType.UPSERT.toString(), ChangeType.DELETE.toString())) + .build()); + return testMCLSideEffect; + } +} diff --git a/metadata-service/plugin/src/test/resources/test-entity-registry-plugins-1.yml b/metadata-service/plugin/src/test/resources/test-entity-registry-plugins-1.yml new file mode 100644 index 0000000000000..903e32b623db0 --- /dev/null +++ b/metadata-service/plugin/src/test/resources/test-entity-registry-plugins-1.yml @@ -0,0 +1,67 @@ +id: test-registry-1 +entities: + - name: dataset + keyAspect: datasetKey + category: core + aspects: + - datasetProperties + - schemaMetadata + - status + - name: chart + keyAspect: chartKey + aspects: + - chartInfo + - status +events: + - name: testEvent + +plugins: + aspectPayloadValidators: + # All status aspects on any entity + - className: 'com.datahub.plugins.metadata.aspect.SpringPluginFactoryTest$TestValidator' + enabled: true + supportedOperations: + - UPSERT + supportedEntityAspectNames: + - entityName: '*' + aspectName: status + # Chart status only + - className: 'com.datahub.plugins.metadata.aspect.SpringPluginFactoryTest$TestValidator' + enabled: true + supportedOperations: + - UPSERT + supportedEntityAspectNames: + - entityName: chart + aspectName: status + # Disabled + - className: 'com.datahub.plugins.metadata.aspect.SpringPluginFactoryTest$TestValidator' + enabled: false + supportedOperations: + - DELETE + supportedEntityAspectNames: + - entityName: '*' + aspectName: status + mutationHooks: + - className: 'com.datahub.plugins.metadata.aspect.SpringPluginFactoryTest$TestMutator' + enabled: true + supportedOperations: + - UPSERT + supportedEntityAspectNames: + - entityName: '*' + aspectName: schemaMetadata + mclSideEffects: + - className: 'com.datahub.plugins.metadata.aspect.SpringPluginFactoryTest$TestMCLSideEffect' + enabled: true + supportedOperations: + - UPSERT + supportedEntityAspectNames: + - entityName: chart + aspectName: chartInfo + mcpSideEffects: + - className: 'com.datahub.plugins.metadata.aspect.SpringPluginFactoryTest$TestMCPSideEffect' + enabled: true + supportedOperations: + - UPSERT + supportedEntityAspectNames: + - entityName: dataset + aspectName: datasetKey diff --git a/metadata-service/plugin/src/test/resources/test-entity-registry-spring-plugins-1.yml b/metadata-service/plugin/src/test/resources/test-entity-registry-spring-plugins-1.yml new file mode 100644 index 0000000000000..a7ae97e9bceb6 --- /dev/null +++ b/metadata-service/plugin/src/test/resources/test-entity-registry-spring-plugins-1.yml @@ -0,0 +1,52 @@ +id: test-registry-spring-1 +entities: [] +plugins: + aspectPayloadValidators: + - className: 'com.datahub.plugins.metadata.aspect.SpringPluginFactoryTest$TestValidator' + packageScan: + - com.datahub.plugins.metadata.aspect.test + spring: + enabled: true + enabled: true + supportedOperations: + - UPSERT + - DELETE + supportedEntityAspectNames: + - entityName: '*' + aspectName: status + mutationHooks: + - className: 'com.datahub.plugins.metadata.aspect.SpringPluginFactoryTest$TestMutator' + packageScan: + - com.datahub.plugins.metadata.aspect.test + spring: + enabled: true + supportedOperations: + - UPSERT + - DELETE + supportedEntityAspectNames: + - entityName: '*' + aspectName: schemaMetadata + mclSideEffects: + - className: 'com.datahub.plugins.metadata.aspect.SpringPluginFactoryTest$TestMCLSideEffect' + packageScan: + - com.datahub.plugins.metadata.aspect.test + spring: + enabled: true + supportedOperations: + - UPSERT + - DELETE + supportedEntityAspectNames: + - entityName: chart + aspectName: chartInfo + mcpSideEffects: + - className: 'com.datahub.plugins.metadata.aspect.SpringPluginFactoryTest$TestMCPSideEffect' + packageScan: + - com.datahub.plugins.metadata.aspect.test + spring: + enabled: true + supportedOperations: + - UPSERT + - DELETE + supportedEntityAspectNames: + - entityName: dataset + aspectName: datasetKey diff --git a/metadata-service/restli-api/src/main/idl/com.linkedin.entity.aspects.restspec.json b/metadata-service/restli-api/src/main/idl/com.linkedin.entity.aspects.restspec.json index 32e7a0e58e535..888d55639d02b 100644 --- a/metadata-service/restli-api/src/main/idl/com.linkedin.entity.aspects.restspec.json +++ b/metadata-service/restli-api/src/main/idl/com.linkedin.entity.aspects.restspec.json @@ -87,6 +87,18 @@ "default" : "unset" } ], "returns" : "string" + }, { + "name" : "ingestProposalBatch", + "javaMethodName" : "ingestProposalBatch", + "parameters" : [ { + "name" : "proposals", + "type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.mxe.MetadataChangeProposal\" }" + }, { + "name" : "async", + "type" : "string", + "default" : "unset" + } ], + "returns" : "string" }, { "name" : "restoreIndices", "javaMethodName" : "restoreIndices", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json index 00b434d30356f..72578be8c54d0 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json @@ -101,6 +101,11 @@ }, "doc" : "Additional properties", "optional" : true + }, { + "name" : "version", + "type" : "string", + "doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.", + "optional" : true } ] } } ] @@ -843,10 +848,10 @@ "PRE" : "Designates pre-production fabrics", "PROD" : "Designates production fabrics", "QA" : "Designates quality assurance fabrics", + "RVW" : "Designates review fabrics", "STG" : "Designates staging fabrics", "TEST" : "Designates testing fabrics", - "UAT" : "Designates user acceptance testing fabrics", - "RVW" : "Designates review fabrics" + "UAT" : "Designates user acceptance testing fabrics" } }, { "type" : "record", @@ -2353,6 +2358,29 @@ "type" : "string", "doc" : "DataHub-native Title, e.g. 'Software Engineer'", "optional" : true + }, { + "name" : "platforms", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "The platforms that the user commonly works with", + "optional" : true, + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataPlatform" ], + "name" : "IsUserOf" + } + } + }, { + "name" : "persona", + "type" : "com.linkedin.common.Urn", + "doc" : "The user's persona type, based on their role", + "optional" : true, + "Relationship" : { + "entityTypes" : [ "dataHubPersona" ], + "name" : "IsPersona" + } }, { "name" : "slack", "type" : "string", @@ -2489,7 +2517,13 @@ }, { "name" : "lastModified", "type" : "com.linkedin.common.AuditStamp", - "doc" : "Audit stamp containing who last modified the status and when." + "doc" : "Audit stamp containing who last modified the status and when.", + "Searchable" : { + "/time" : { + "fieldName" : "statusLastModifiedAt", + "fieldType" : "COUNT" + } + } } ], "Aspect" : { "name" : "corpUserStatus" @@ -2861,8 +2895,9 @@ }, { "name" : "label", "type" : "string", - "doc" : "Label of the field. Provides a more human-readable name for the field than field path. Some sources will\nprovide this metadata but not all sources have the concept of a label. If just one string is associated with\na field in a source, that is most likely a description.", + "doc" : "Label of the field. Provides a more human-readable name for the field than field path. Some sources will\nprovide this metadata but not all sources have the concept of a label. If just one string is associated with\na field in a source, that is most likely a description.\n\nNote that this field is deprecated and is not surfaced in the UI.", "optional" : true, + "Deprecated" : true, "Searchable" : { "boostScore" : 0.2, "fieldName" : "fieldLabels", @@ -3287,6 +3322,7 @@ "name" : "MLFeatureProperties", "namespace" : "com.linkedin.ml.metadata", "doc" : "Properties associated with a MLFeature", + "include" : [ "com.linkedin.common.CustomProperties" ], "fields" : [ { "name" : "description", "type" : "string", @@ -4021,7 +4057,15 @@ }, { "name" : "systemMetadata", "type" : "SystemMetadata", - "doc" : "A string->string map of custom properties that one might want to attach to an event\n", + "doc" : "System properties that one might want to attach to an event\n", + "optional" : true + }, { + "name" : "headers", + "type" : { + "type" : "map", + "values" : "string" + }, + "doc" : "Headers - intended to mimic http headers", "optional" : true } ] }, "com.linkedin.mxe.SystemMetadata", "com.linkedin.schema.ArrayType", "com.linkedin.schema.BinaryJsonSchema", "com.linkedin.schema.BooleanType", "com.linkedin.schema.BytesType", "com.linkedin.schema.DatasetFieldForeignKey", "com.linkedin.schema.DateType", "com.linkedin.schema.EditableSchemaFieldInfo", "com.linkedin.schema.EditableSchemaMetadata", "com.linkedin.schema.EnumType", "com.linkedin.schema.EspressoSchema", "com.linkedin.schema.FixedType", "com.linkedin.schema.ForeignKeyConstraint", "com.linkedin.schema.ForeignKeySpec", "com.linkedin.schema.KafkaSchema", "com.linkedin.schema.KeyValueSchema", "com.linkedin.schema.MapType", "com.linkedin.schema.MySqlDDL", "com.linkedin.schema.NullType", "com.linkedin.schema.NumberType", "com.linkedin.schema.OracleDDL", "com.linkedin.schema.OrcSchema", "com.linkedin.schema.OtherSchema", "com.linkedin.schema.PrestoDDL", "com.linkedin.schema.RecordType", "com.linkedin.schema.SchemaField", "com.linkedin.schema.SchemaFieldDataType", "com.linkedin.schema.SchemaMetadata", "com.linkedin.schema.SchemaMetadataKey", "com.linkedin.schema.Schemaless", "com.linkedin.schema.StringType", "com.linkedin.schema.TimeType", "com.linkedin.schema.UnionType", "com.linkedin.schema.UrnForeignKey", "com.linkedin.tag.TagProperties" ], @@ -4114,6 +4158,18 @@ "default" : "unset" } ], "returns" : "string" + }, { + "name" : "ingestProposalBatch", + "javaMethodName" : "ingestProposalBatch", + "parameters" : [ { + "name" : "proposals", + "type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.mxe.MetadataChangeProposal\" }" + }, { + "name" : "async", + "type" : "string", + "default" : "unset" + } ], + "returns" : "string" }, { "name" : "restoreIndices", "javaMethodName" : "restoreIndices", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index ffbcdd1b2adb3..9b93f1184cd59 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -843,10 +843,10 @@ "PRE" : "Designates pre-production fabrics", "PROD" : "Designates production fabrics", "QA" : "Designates quality assurance fabrics", + "RVW" : "Designates review fabrics", "STG" : "Designates staging fabrics", "TEST" : "Designates testing fabrics", - "UAT" : "Designates user acceptance testing fabrics", - "RVW" : "Designates review fabrics" + "UAT" : "Designates user acceptance testing fabrics" } }, { "type" : "record", @@ -2766,6 +2766,29 @@ "type" : "string", "doc" : "DataHub-native Title, e.g. 'Software Engineer'", "optional" : true + }, { + "name" : "platforms", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "The platforms that the user commonly works with", + "optional" : true, + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataPlatform" ], + "name" : "IsUserOf" + } + } + }, { + "name" : "persona", + "type" : "com.linkedin.common.Urn", + "doc" : "The user's persona type, based on their role", + "optional" : true, + "Relationship" : { + "entityTypes" : [ "dataHubPersona" ], + "name" : "IsPersona" + } }, { "name" : "slack", "type" : "string", @@ -2801,7 +2824,13 @@ }, { "name" : "lastModified", "type" : "com.linkedin.common.AuditStamp", - "doc" : "Audit stamp containing who last modified the status and when." + "doc" : "Audit stamp containing who last modified the status and when.", + "Searchable" : { + "/time" : { + "fieldName" : "statusLastModifiedAt", + "fieldType" : "COUNT" + } + } } ], "Aspect" : { "name" : "corpUserStatus" @@ -3249,8 +3278,9 @@ }, { "name" : "label", "type" : "string", - "doc" : "Label of the field. Provides a more human-readable name for the field than field path. Some sources will\nprovide this metadata but not all sources have the concept of a label. If just one string is associated with\na field in a source, that is most likely a description.", + "doc" : "Label of the field. Provides a more human-readable name for the field than field path. Some sources will\nprovide this metadata but not all sources have the concept of a label. If just one string is associated with\na field in a source, that is most likely a description.\n\nNote that this field is deprecated and is not surfaced in the UI.", "optional" : true, + "Deprecated" : true, "Searchable" : { "boostScore" : 0.2, "fieldName" : "fieldLabels", @@ -4364,6 +4394,7 @@ "name" : "MLPrimaryKeyProperties", "namespace" : "com.linkedin.ml.metadata", "doc" : "Properties associated with a MLPrimaryKey", + "include" : [ "com.linkedin.common.CustomProperties" ], "fields" : [ { "name" : "description", "type" : "string", @@ -4453,6 +4484,7 @@ "name" : "MLFeatureProperties", "namespace" : "com.linkedin.ml.metadata", "doc" : "Properties associated with a MLFeature", + "include" : [ "com.linkedin.common.CustomProperties" ], "fields" : [ { "name" : "description", "type" : "string", @@ -5344,7 +5376,12 @@ "items" : "com.linkedin.common.Urn" }, "doc" : "A specific set of users to apply the policy to (disjunctive)", - "optional" : true + "optional" : true, + "Searchable" : { + "/*" : { + "fieldType" : "URN" + } + } }, { "name" : "groups", "type" : { @@ -5352,7 +5389,12 @@ "items" : "com.linkedin.common.Urn" }, "doc" : "A specific set of groups to apply the policy to (disjunctive)", - "optional" : true + "optional" : true, + "Searchable" : { + "/*" : { + "fieldType" : "URN" + } + } }, { "name" : "resourceOwners", "type" : "boolean", @@ -5370,12 +5412,18 @@ "name" : "allUsers", "type" : "boolean", "doc" : "Whether the filter should apply to all users.", - "default" : false + "default" : false, + "Searchable" : { + "fieldType" : "BOOLEAN" + } }, { "name" : "allGroups", "type" : "boolean", "doc" : "Whether the filter should apply to all groups.", - "default" : false + "default" : false, + "Searchable" : { + "fieldType" : "BOOLEAN" + } }, { "name" : "roles", "type" : { @@ -5389,6 +5437,11 @@ "entityTypes" : [ "dataHubRole" ], "name" : "IsAssociatedWithRole" } + }, + "Searchable" : { + "/*" : { + "fieldType" : "URN" + } } } ] }, @@ -6211,6 +6264,11 @@ "type" : "boolean", "doc" : "Marks an entity as having been explored for as a part of the graph walk", "optional" : true + }, { + "name" : "truncatedChildren", + "type" : "boolean", + "doc" : "Indicates this destination node has additional unexplored child relationships", + "optional" : true }, { "name" : "ignoredAsHop", "type" : "boolean", @@ -6411,6 +6469,11 @@ }, "doc" : "Additional properties", "optional" : true + }, { + "name" : "version", + "type" : "string", + "doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.", + "optional" : true } ] }, "com.linkedin.policy.DataHubActorFilter", "com.linkedin.policy.DataHubPolicyInfo", "com.linkedin.policy.DataHubResourceFilter", "com.linkedin.policy.PolicyMatchCondition", "com.linkedin.policy.PolicyMatchCriterion", "com.linkedin.policy.PolicyMatchFilter", "com.linkedin.retention.DataHubRetentionConfig", "com.linkedin.retention.Retention", "com.linkedin.retention.TimeBasedRetention", "com.linkedin.retention.VersionBasedRetention", "com.linkedin.schema.ArrayType", "com.linkedin.schema.BinaryJsonSchema", "com.linkedin.schema.BooleanType", "com.linkedin.schema.BytesType", "com.linkedin.schema.DatasetFieldForeignKey", "com.linkedin.schema.DateType", "com.linkedin.schema.EditableSchemaFieldInfo", "com.linkedin.schema.EditableSchemaMetadata", "com.linkedin.schema.EnumType", "com.linkedin.schema.EspressoSchema", "com.linkedin.schema.FixedType", "com.linkedin.schema.ForeignKeyConstraint", "com.linkedin.schema.ForeignKeySpec", "com.linkedin.schema.KafkaSchema", "com.linkedin.schema.KeyValueSchema", "com.linkedin.schema.MapType", "com.linkedin.schema.MySqlDDL", "com.linkedin.schema.NullType", "com.linkedin.schema.NumberType", "com.linkedin.schema.OracleDDL", "com.linkedin.schema.OrcSchema", "com.linkedin.schema.OtherSchema", "com.linkedin.schema.PrestoDDL", "com.linkedin.schema.RecordType", "com.linkedin.schema.SchemaField", "com.linkedin.schema.SchemaFieldDataType", "com.linkedin.schema.SchemaMetadata", "com.linkedin.schema.SchemaMetadataKey", "com.linkedin.schema.Schemaless", "com.linkedin.schema.StringType", "com.linkedin.schema.TimeType", "com.linkedin.schema.UnionType", "com.linkedin.schema.UrnForeignKey", "com.linkedin.tag.TagProperties" ], "schema" : { diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json index 3eac87e268f5d..9bf7f97b34be1 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json @@ -146,6 +146,11 @@ }, "doc" : "Additional properties", "optional" : true + }, { + "name" : "version", + "type" : "string", + "doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.", + "optional" : true } ] }, "doc" : "The system metadata for this aspect\n", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json index 1733537e68f30..ef72ecb23d5bb 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json @@ -155,6 +155,11 @@ }, "doc" : "Additional properties", "optional" : true + }, { + "name" : "version", + "type" : "string", + "doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.", + "optional" : true } ] }, "doc" : "The system metadata for this aspect\n", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json index 0139072b2ae15..18ef55011ed5a 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json @@ -585,10 +585,10 @@ "PRE" : "Designates pre-production fabrics", "PROD" : "Designates production fabrics", "QA" : "Designates quality assurance fabrics", + "RVW" : "Designates review fabrics", "STG" : "Designates staging fabrics", "TEST" : "Designates testing fabrics", - "UAT" : "Designates user acceptance testing fabrics", - "RVW" : "Designates review fabrics" + "UAT" : "Designates user acceptance testing fabrics" } }, { "type" : "record", @@ -2086,6 +2086,29 @@ "type" : "string", "doc" : "DataHub-native Title, e.g. 'Software Engineer'", "optional" : true + }, { + "name" : "platforms", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "The platforms that the user commonly works with", + "optional" : true, + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataPlatform" ], + "name" : "IsUserOf" + } + } + }, { + "name" : "persona", + "type" : "com.linkedin.common.Urn", + "doc" : "The user's persona type, based on their role", + "optional" : true, + "Relationship" : { + "entityTypes" : [ "dataHubPersona" ], + "name" : "IsPersona" + } }, { "name" : "slack", "type" : "string", @@ -2222,7 +2245,13 @@ }, { "name" : "lastModified", "type" : "com.linkedin.common.AuditStamp", - "doc" : "Audit stamp containing who last modified the status and when." + "doc" : "Audit stamp containing who last modified the status and when.", + "Searchable" : { + "/time" : { + "fieldName" : "statusLastModifiedAt", + "fieldType" : "COUNT" + } + } } ], "Aspect" : { "name" : "corpUserStatus" @@ -2594,8 +2623,9 @@ }, { "name" : "label", "type" : "string", - "doc" : "Label of the field. Provides a more human-readable name for the field than field path. Some sources will\nprovide this metadata but not all sources have the concept of a label. If just one string is associated with\na field in a source, that is most likely a description.", + "doc" : "Label of the field. Provides a more human-readable name for the field than field path. Some sources will\nprovide this metadata but not all sources have the concept of a label. If just one string is associated with\na field in a source, that is most likely a description.\n\nNote that this field is deprecated and is not surfaced in the UI.", "optional" : true, + "Deprecated" : true, "Searchable" : { "boostScore" : 0.2, "fieldName" : "fieldLabels", @@ -3020,6 +3050,7 @@ "name" : "MLFeatureProperties", "namespace" : "com.linkedin.ml.metadata", "doc" : "Properties associated with a MLFeature", + "include" : [ "com.linkedin.common.CustomProperties" ], "fields" : [ { "name" : "description", "type" : "string", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json index 3886faffadedb..ba29f43dae0a6 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json @@ -182,6 +182,11 @@ "type" : "boolean", "doc" : "Marks this relationship as explored during the graph walk", "optional" : true + }, { + "name" : "truncatedChildren", + "type" : "boolean", + "doc" : "Indicates this destination node has additional unexplored child relationships", + "optional" : true }, { "name" : "ignoredAsHop", "type" : "boolean", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json index 1caeed2570317..cf05978820911 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json @@ -585,10 +585,10 @@ "PRE" : "Designates pre-production fabrics", "PROD" : "Designates production fabrics", "QA" : "Designates quality assurance fabrics", + "RVW" : "Designates review fabrics", "STG" : "Designates staging fabrics", "TEST" : "Designates testing fabrics", - "UAT" : "Designates user acceptance testing fabrics", - "RVW" : "Designates review fabrics" + "UAT" : "Designates user acceptance testing fabrics" } }, { "type" : "record", @@ -2080,6 +2080,29 @@ "type" : "string", "doc" : "DataHub-native Title, e.g. 'Software Engineer'", "optional" : true + }, { + "name" : "platforms", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "The platforms that the user commonly works with", + "optional" : true, + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataPlatform" ], + "name" : "IsUserOf" + } + } + }, { + "name" : "persona", + "type" : "com.linkedin.common.Urn", + "doc" : "The user's persona type, based on their role", + "optional" : true, + "Relationship" : { + "entityTypes" : [ "dataHubPersona" ], + "name" : "IsPersona" + } }, { "name" : "slack", "type" : "string", @@ -2216,7 +2239,13 @@ }, { "name" : "lastModified", "type" : "com.linkedin.common.AuditStamp", - "doc" : "Audit stamp containing who last modified the status and when." + "doc" : "Audit stamp containing who last modified the status and when.", + "Searchable" : { + "/time" : { + "fieldName" : "statusLastModifiedAt", + "fieldType" : "COUNT" + } + } } ], "Aspect" : { "name" : "corpUserStatus" @@ -2588,8 +2617,9 @@ }, { "name" : "label", "type" : "string", - "doc" : "Label of the field. Provides a more human-readable name for the field than field path. Some sources will\nprovide this metadata but not all sources have the concept of a label. If just one string is associated with\na field in a source, that is most likely a description.", + "doc" : "Label of the field. Provides a more human-readable name for the field than field path. Some sources will\nprovide this metadata but not all sources have the concept of a label. If just one string is associated with\na field in a source, that is most likely a description.\n\nNote that this field is deprecated and is not surfaced in the UI.", "optional" : true, + "Deprecated" : true, "Searchable" : { "boostScore" : 0.2, "fieldName" : "fieldLabels", @@ -3014,6 +3044,7 @@ "name" : "MLFeatureProperties", "namespace" : "com.linkedin.ml.metadata", "doc" : "Properties associated with a MLFeature", + "include" : [ "com.linkedin.common.CustomProperties" ], "fields" : [ { "name" : "description", "type" : "string", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json index 1592333988b4c..15f16dd2ea6cd 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json @@ -843,10 +843,10 @@ "PRE" : "Designates pre-production fabrics", "PROD" : "Designates production fabrics", "QA" : "Designates quality assurance fabrics", + "RVW" : "Designates review fabrics", "STG" : "Designates staging fabrics", "TEST" : "Designates testing fabrics", - "UAT" : "Designates user acceptance testing fabrics", - "RVW" : "Designates review fabrics" + "UAT" : "Designates user acceptance testing fabrics" } }, { "type" : "record", @@ -2760,6 +2760,29 @@ "type" : "string", "doc" : "DataHub-native Title, e.g. 'Software Engineer'", "optional" : true + }, { + "name" : "platforms", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "The platforms that the user commonly works with", + "optional" : true, + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataPlatform" ], + "name" : "IsUserOf" + } + } + }, { + "name" : "persona", + "type" : "com.linkedin.common.Urn", + "doc" : "The user's persona type, based on their role", + "optional" : true, + "Relationship" : { + "entityTypes" : [ "dataHubPersona" ], + "name" : "IsPersona" + } }, { "name" : "slack", "type" : "string", @@ -2795,7 +2818,13 @@ }, { "name" : "lastModified", "type" : "com.linkedin.common.AuditStamp", - "doc" : "Audit stamp containing who last modified the status and when." + "doc" : "Audit stamp containing who last modified the status and when.", + "Searchable" : { + "/time" : { + "fieldName" : "statusLastModifiedAt", + "fieldType" : "COUNT" + } + } } ], "Aspect" : { "name" : "corpUserStatus" @@ -3243,8 +3272,9 @@ }, { "name" : "label", "type" : "string", - "doc" : "Label of the field. Provides a more human-readable name for the field than field path. Some sources will\nprovide this metadata but not all sources have the concept of a label. If just one string is associated with\na field in a source, that is most likely a description.", + "doc" : "Label of the field. Provides a more human-readable name for the field than field path. Some sources will\nprovide this metadata but not all sources have the concept of a label. If just one string is associated with\na field in a source, that is most likely a description.\n\nNote that this field is deprecated and is not surfaced in the UI.", "optional" : true, + "Deprecated" : true, "Searchable" : { "boostScore" : 0.2, "fieldName" : "fieldLabels", @@ -4358,6 +4388,7 @@ "name" : "MLPrimaryKeyProperties", "namespace" : "com.linkedin.ml.metadata", "doc" : "Properties associated with a MLPrimaryKey", + "include" : [ "com.linkedin.common.CustomProperties" ], "fields" : [ { "name" : "description", "type" : "string", @@ -4447,6 +4478,7 @@ "name" : "MLFeatureProperties", "namespace" : "com.linkedin.ml.metadata", "doc" : "Properties associated with a MLFeature", + "include" : [ "com.linkedin.common.CustomProperties" ], "fields" : [ { "name" : "description", "type" : "string", @@ -5338,7 +5370,12 @@ "items" : "com.linkedin.common.Urn" }, "doc" : "A specific set of users to apply the policy to (disjunctive)", - "optional" : true + "optional" : true, + "Searchable" : { + "/*" : { + "fieldType" : "URN" + } + } }, { "name" : "groups", "type" : { @@ -5346,7 +5383,12 @@ "items" : "com.linkedin.common.Urn" }, "doc" : "A specific set of groups to apply the policy to (disjunctive)", - "optional" : true + "optional" : true, + "Searchable" : { + "/*" : { + "fieldType" : "URN" + } + } }, { "name" : "resourceOwners", "type" : "boolean", @@ -5364,12 +5406,18 @@ "name" : "allUsers", "type" : "boolean", "doc" : "Whether the filter should apply to all users.", - "default" : false + "default" : false, + "Searchable" : { + "fieldType" : "BOOLEAN" + } }, { "name" : "allGroups", "type" : "boolean", "doc" : "Whether the filter should apply to all groups.", - "default" : false + "default" : false, + "Searchable" : { + "fieldType" : "BOOLEAN" + } }, { "name" : "roles", "type" : { @@ -5383,6 +5431,11 @@ "entityTypes" : [ "dataHubRole" ], "name" : "IsAssociatedWithRole" } + }, + "Searchable" : { + "/*" : { + "fieldType" : "URN" + } } } ] }, diff --git a/metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClient.java b/metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClient.java index a7980d0d5c99f..8821143cde6cc 100644 --- a/metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClient.java +++ b/metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClient.java @@ -1,6 +1,7 @@ package com.linkedin.entity.client; import static com.linkedin.metadata.utils.GenericRecordUtils.entityResponseToAspectMap; +import static com.linkedin.metadata.utils.GenericRecordUtils.entityResponseToSystemAspectMap; import com.datahub.plugins.auth.authorization.Authorizer; import com.linkedin.common.VersionedUrn; @@ -12,6 +13,7 @@ import com.linkedin.entity.Entity; import com.linkedin.entity.EntityResponse; import com.linkedin.metadata.aspect.EnvelopedAspect; +import com.linkedin.metadata.aspect.SystemAspect; import com.linkedin.metadata.aspect.VersionedAspect; import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.browse.BrowseResultV2; @@ -602,4 +604,13 @@ default Map> getLatestAspects( String entityName = urns.stream().findFirst().map(Urn::getEntityType).get(); return entityResponseToAspectMap(batchGetV2(opContext, entityName, urns, aspectNames)); } + + @Nonnull + default Map> getLatestSystemAspect( + @Nonnull OperationContext opContext, @Nonnull Set urns, @Nonnull Set aspectNames) + throws RemoteInvocationException, URISyntaxException { + String entityName = urns.stream().findFirst().map(Urn::getEntityType).get(); + return entityResponseToSystemAspectMap( + batchGetV2(opContext, entityName, urns, aspectNames), opContext.getEntityRegistry()); + } } diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java index 70fae208ad77a..fe1ca571efea5 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java @@ -85,8 +85,13 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.mail.MethodNotSupportedException; @@ -110,14 +115,17 @@ public class RestliEntityClient extends BaseClient implements EntityClient { private static final RunsRequestBuilders RUNS_REQUEST_BUILDERS = new RunsRequestBuilders(); private final int batchGetV2Size; + private final int batchGetV2Concurrency; public RestliEntityClient( @Nonnull final Client restliClient, @Nonnull final BackoffPolicy backoffPolicy, int retryCount, - int batchGetV2Size) { + int batchGetV2Size, + int batchGetV2Concurrency) { super(restliClient, backoffPolicy, retryCount); this.batchGetV2Size = Math.max(1, batchGetV2Size); + this.batchGetV2Concurrency = batchGetV2Concurrency; } @Override @@ -150,7 +158,6 @@ public Entity get(@Nonnull OperationContext opContext, @Nonnull final Urn urn) *

Batch get a set of {@link Entity} objects by urn. * * @param urns the urns of the entities to batch get - * @param authentication the authentication to include in the request to the Metadata Service * @throws RemoteInvocationException when unable to execute request */ @Override @@ -216,40 +223,54 @@ public Map batchGetV2( throws RemoteInvocationException, URISyntaxException { Map responseMap = new HashMap<>(); + ExecutorService executor = Executors.newFixedThreadPool(Math.max(1, batchGetV2Concurrency)); - Iterators.partition(urns.iterator(), batchGetV2Size) - .forEachRemaining( - batch -> { - try { - final EntitiesV2BatchGetRequestBuilder requestBuilder = - ENTITIES_V2_REQUEST_BUILDERS - .batchGet() - .aspectsParam(aspectNames) - .ids(batch.stream().map(Urn::toString).collect(Collectors.toList())); - - responseMap.putAll( - sendClientRequest(requestBuilder, opContext.getSessionAuthentication()) - .getEntity() - .getResults() - .entrySet() - .stream() - .collect( - Collectors.toMap( - entry -> { - try { - return Urn.createFromString(entry.getKey()); - } catch (URISyntaxException e) { - throw new RuntimeException( - String.format( - "Failed to bind urn string with value %s into urn", - entry.getKey())); - } - }, - entry -> entry.getValue().getEntity()))); - } catch (RemoteInvocationException e) { - throw new RuntimeException(e); - } - }); + try { + Iterable> iterable = () -> Iterators.partition(urns.iterator(), batchGetV2Size); + List>> futures = + StreamSupport.stream(iterable.spliterator(), false) + .map( + batch -> + executor.submit( + () -> { + try { + log.debug("Executing batchGetV2 with batch size: {}", batch.size()); + final EntitiesV2BatchGetRequestBuilder requestBuilder = + ENTITIES_V2_REQUEST_BUILDERS + .batchGet() + .aspectsParam(aspectNames) + .ids( + batch.stream() + .map(Urn::toString) + .collect(Collectors.toList())); + + return sendClientRequest( + requestBuilder, opContext.getSessionAuthentication()) + .getEntity() + .getResults() + .entrySet() + .stream() + .collect( + Collectors.toMap( + entry -> UrnUtils.getUrn(entry.getKey()), + entry -> entry.getValue().getEntity())); + } catch (RemoteInvocationException e) { + throw new RuntimeException(e); + } + })) + .collect(Collectors.toList()); + + futures.forEach( + result -> { + try { + responseMap.putAll(result.get()); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }); + } finally { + executor.shutdown(); + } return responseMap; } @@ -260,7 +281,6 @@ public Map batchGetV2( * @param entityName the entity type to fetch * @param versionedUrns the urns of the entities to batch get * @param aspectNames the aspect names to batch get - * @param authentication the authentication to include in the request to the Metadata Service * @throws RemoteInvocationException when unable to execute request */ @Override @@ -272,39 +292,62 @@ public Map batchGetVersionedV2( @Nullable final Set aspectNames) { Map responseMap = new HashMap<>(); + ExecutorService executor = Executors.newFixedThreadPool(Math.max(1, batchGetV2Concurrency)); - Iterators.partition(versionedUrns.iterator(), batchGetV2Size) - .forEachRemaining( - batch -> { - final EntitiesVersionedV2BatchGetRequestBuilder requestBuilder = - ENTITIES_VERSIONED_V2_REQUEST_BUILDERS - .batchGet() - .aspectsParam(aspectNames) - .entityTypeParam(entityName) - .ids( - batch.stream() - .map( - versionedUrn -> - com.linkedin.common.urn.VersionedUrn.of( - versionedUrn.getUrn().toString(), - versionedUrn.getVersionStamp())) - .collect(Collectors.toSet())); - - try { - responseMap.putAll( - sendClientRequest(requestBuilder, opContext.getSessionAuthentication()) - .getEntity() - .getResults() - .entrySet() - .stream() - .collect( - Collectors.toMap( - entry -> UrnUtils.getUrn(entry.getKey().getUrn()), - entry -> entry.getValue().getEntity()))); - } catch (RemoteInvocationException e) { - throw new RuntimeException(e); - } - }); + try { + Iterable> iterable = + () -> Iterators.partition(versionedUrns.iterator(), batchGetV2Size); + List>> futures = + StreamSupport.stream(iterable.spliterator(), false) + .map( + batch -> + executor.submit( + () -> { + try { + log.debug( + "Executing batchGetVersionedV2 with batch size: {}", + batch.size()); + final EntitiesVersionedV2BatchGetRequestBuilder requestBuilder = + ENTITIES_VERSIONED_V2_REQUEST_BUILDERS + .batchGet() + .aspectsParam(aspectNames) + .entityTypeParam(entityName) + .ids( + batch.stream() + .map( + versionedUrn -> + com.linkedin.common.urn.VersionedUrn.of( + versionedUrn.getUrn().toString(), + versionedUrn.getVersionStamp())) + .collect(Collectors.toSet())); + + return sendClientRequest( + requestBuilder, opContext.getSessionAuthentication()) + .getEntity() + .getResults() + .entrySet() + .stream() + .collect( + Collectors.toMap( + entry -> UrnUtils.getUrn(entry.getKey().getUrn()), + entry -> entry.getValue().getEntity())); + } catch (RemoteInvocationException e) { + throw new RuntimeException(e); + } + })) + .collect(Collectors.toList()); + + futures.forEach( + result -> { + try { + responseMap.putAll(result.get()); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }); + } finally { + executor.shutdown(); + } return responseMap; } @@ -955,7 +998,6 @@ public VersionedAspect getAspectOrNull( * @param startTimeMillis the earliest desired event time of the aspect value in milliseconds. * @param endTimeMillis the latest desired event time of the aspect value in milliseconds. * @param limit the maximum number of desired aspect values. - * @param authentication the actor associated with the request [internal] * @return the list of EnvelopedAspect values satisfying the input parameters. * @throws RemoteInvocationException on remote request error. */ diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemRestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemRestliEntityClient.java index 364ee9b0519d2..7546d1f0a3b54 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemRestliEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemRestliEntityClient.java @@ -27,8 +27,9 @@ public SystemRestliEntityClient( @Nonnull final BackoffPolicy backoffPolicy, int retryCount, EntityClientCacheConfig cacheConfig, - int batchGetV2Size) { - super(restliClient, backoffPolicy, retryCount, batchGetV2Size); + int batchGetV2Size, + int batchGetV2Concurrency) { + super(restliClient, backoffPolicy, retryCount, batchGetV2Size, batchGetV2Concurrency); this.operationContextMap = CacheBuilder.newBuilder().maximumSize(500).build(); this.entityClientCache = buildEntityClientCache(SystemRestliEntityClient.class, cacheConfig); } diff --git a/metadata-service/restli-client/src/test/java/com/linkedin/common/client/BaseClientTest.java b/metadata-service/restli-client/src/test/java/com/linkedin/common/client/BaseClientTest.java index 474bb24f9e16b..797ead10c1a66 100644 --- a/metadata-service/restli-client/src/test/java/com/linkedin/common/client/BaseClientTest.java +++ b/metadata-service/restli-client/src/test/java/com/linkedin/common/client/BaseClientTest.java @@ -37,7 +37,7 @@ public void testZeroRetry() throws RemoteInvocationException { when(mockRestliClient.sendRequest(any(ActionRequest.class))).thenReturn(mockFuture); RestliEntityClient testClient = - new RestliEntityClient(mockRestliClient, new ExponentialBackoff(1), 0, 10); + new RestliEntityClient(mockRestliClient, new ExponentialBackoff(1), 0, 10, 2); testClient.sendClientRequest(testRequestBuilder, AUTH); // Expected 1 actual try and 0 retries verify(mockRestliClient).sendRequest(any(ActionRequest.class)); @@ -56,7 +56,7 @@ public void testMultipleRetries() throws RemoteInvocationException { .thenReturn(mockFuture); RestliEntityClient testClient = - new RestliEntityClient(mockRestliClient, new ExponentialBackoff(1), 1, 10); + new RestliEntityClient(mockRestliClient, new ExponentialBackoff(1), 1, 10, 2); testClient.sendClientRequest(testRequestBuilder, AUTH); // Expected 1 actual try and 1 retries verify(mockRestliClient, times(2)).sendRequest(any(ActionRequest.class)); @@ -73,7 +73,7 @@ public void testNonRetry() { .thenThrow(new RuntimeException(new RequiredFieldNotPresentException("value"))); RestliEntityClient testClient = - new RestliEntityClient(mockRestliClient, new ExponentialBackoff(1), 1, 10); + new RestliEntityClient(mockRestliClient, new ExponentialBackoff(1), 1, 10, 2); assertThrows( RuntimeException.class, () -> testClient.sendClientRequest(testRequestBuilder, AUTH)); } diff --git a/metadata-service/restli-client/src/test/java/com/linkedin/entity/client/SystemRestliEntityClientTest.java b/metadata-service/restli-client/src/test/java/com/linkedin/entity/client/SystemRestliEntityClientTest.java index 75614ca998f6a..e6d53fc98e2e3 100644 --- a/metadata-service/restli-client/src/test/java/com/linkedin/entity/client/SystemRestliEntityClientTest.java +++ b/metadata-service/restli-client/src/test/java/com/linkedin/entity/client/SystemRestliEntityClientTest.java @@ -45,7 +45,8 @@ public void testCache() throws RemoteInvocationException, URISyntaxException { noCacheConfig.setEnabled(true); SystemRestliEntityClient noCacheTest = - new SystemRestliEntityClient(mockRestliClient, new ConstantBackoff(0), 0, noCacheConfig, 1); + new SystemRestliEntityClient( + mockRestliClient, new ConstantBackoff(0), 0, noCacheConfig, 1, 2); com.linkedin.entity.EntityResponse responseStatusTrue = buildStatusResponse(true); com.linkedin.entity.EntityResponse responseStatusFalse = buildStatusResponse(false); @@ -83,7 +84,8 @@ public void testCache() throws RemoteInvocationException, URISyntaxException { Map.of(TEST_URN.getEntityType(), Map.of(Constants.STATUS_ASPECT_NAME, 60))); SystemRestliEntityClient cacheTest = - new SystemRestliEntityClient(mockRestliClient, new ConstantBackoff(0), 0, cacheConfig, 1); + new SystemRestliEntityClient( + mockRestliClient, new ConstantBackoff(0), 0, cacheConfig, 1, 2); mockResponse(mockRestliClient, responseStatusTrue); assertEquals( @@ -117,7 +119,8 @@ public void testBatchCache() throws RemoteInvocationException, URISyntaxExceptio noCacheConfig.setEnabled(true); SystemRestliEntityClient noCacheTest = - new SystemRestliEntityClient(mockRestliClient, new ConstantBackoff(0), 0, noCacheConfig, 1); + new SystemRestliEntityClient( + mockRestliClient, new ConstantBackoff(0), 0, noCacheConfig, 1, 2); com.linkedin.entity.EntityResponse responseStatusTrue = buildStatusResponse(true); com.linkedin.entity.EntityResponse responseStatusFalse = buildStatusResponse(false); @@ -155,7 +158,8 @@ public void testBatchCache() throws RemoteInvocationException, URISyntaxExceptio Map.of(TEST_URN.getEntityType(), Map.of(Constants.STATUS_ASPECT_NAME, 60))); SystemRestliEntityClient cacheTest = - new SystemRestliEntityClient(mockRestliClient, new ConstantBackoff(0), 0, cacheConfig, 1); + new SystemRestliEntityClient( + mockRestliClient, new ConstantBackoff(0), 0, cacheConfig, 1, 2); mockResponse(mockRestliClient, responseStatusTrue); assertEquals( diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/filter/RestliLoggingFilter.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/filter/RestliLoggingFilter.java index edd8270e87210..eee64b3f24bb9 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/filter/RestliLoggingFilter.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/filter/RestliLoggingFilter.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.filter; +import com.linkedin.metadata.restli.NonExceptionHttpErrorResponse; import com.linkedin.restli.common.HttpMethod; import com.linkedin.restli.common.HttpStatus; import com.linkedin.restli.server.filter.Filter; @@ -33,7 +34,9 @@ public CompletableFuture onError( final FilterRequestContext requestContext, final FilterResponseContext responseContext) { logResponse(requestContext, responseContext); - log.error("Rest.li error: ", th); + if (!(th instanceof NonExceptionHttpErrorResponse)) { + log.error("Rest.li error: ", th); + } return CompletableFuture.completedFuture(null); } diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/analytics/Analytics.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/analytics/Analytics.java index 7fcaf82e09096..753dd9b807fd1 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/analytics/Analytics.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/analytics/Analytics.java @@ -14,6 +14,7 @@ import com.linkedin.restli.server.RestLiServiceException; import com.linkedin.restli.server.annotations.Action; import com.linkedin.restli.server.annotations.ActionParam; +import com.linkedin.restli.server.annotations.Context; import com.linkedin.restli.server.annotations.Optional; import com.linkedin.restli.server.annotations.RestLiSimpleResource; import com.linkedin.restli.server.resources.SimpleResourceTemplate; @@ -28,6 +29,7 @@ import javax.annotation.Nullable; import javax.inject.Inject; import javax.inject.Named; +import javax.servlet.http.HttpServletRequest; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.metadata.context.RequestContext; @@ -80,7 +82,8 @@ public Task getTimeseriesStats( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to get entity " + entityName); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_GET_TIMESERIES_STATS, entityName), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), + ACTION_GET_TIMESERIES_STATS, entityName), authorizer, auth, true); log.info("Attempting to query timeseries stats"); GetTimeseriesAggregatedStatsResponse resp = new GetTimeseriesAggregatedStatsResponse(); diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java index 606c479cd14b5..cbca464d569a8 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java @@ -13,7 +13,6 @@ import com.codahale.metrics.MetricRegistry; import com.datahub.authentication.Authentication; import com.datahub.authentication.AuthenticationContext; -import com.datahub.authorization.EntitySpec; import com.datahub.plugins.auth.authorization.Authorizer; import com.google.common.annotations.VisibleForTesting; import com.linkedin.aspect.GetTimeseriesAspectValuesResponse; @@ -21,7 +20,6 @@ import com.linkedin.common.urn.Urn; import com.linkedin.metadata.aspect.EnvelopedAspectArray; import com.linkedin.metadata.aspect.VersionedAspect; -import com.linkedin.metadata.authorization.Disjunctive; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.IngestResult; @@ -53,6 +51,7 @@ import io.opentelemetry.extension.annotations.WithSpan; import java.net.URISyntaxException; import java.time.Clock; +import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -69,10 +68,12 @@ public class AspectResource extends CollectionResourceTaskTemplate get( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to get aspect for " + urn); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli("authorizerChain", urn.getEntityType()), _authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), "authorizerChain", urn.getEntityType()), _authorizer, auth, true); final VersionedAspect aspect = _entityService.getVersionedAspect(opContext, urn, aspectName, version); if (aspect == null) { - throw RestliUtil.resourceNotFoundException( - String.format( - "Did not find urn: %s aspect: %s version: %s", urn, aspectName, version)); + log.warn("Did not find urn: {} aspect: {} version: {}", urn, aspectName, version); + throw RestliUtil.nonExceptionResourceNotFound(); } return new AnyRecord(aspect.data()); }, @@ -196,7 +196,7 @@ public Task getTimeseriesAspectValues( "User is unauthorized to get timeseries aspect for " + urn); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_GET_TIMESERIES_ASPECT, urn.getEntityType()), _authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_GET_TIMESERIES_ASPECT, urn.getEntityType()), _authorizer, auth, true); GetTimeseriesAspectValuesResponse response = new GetTimeseriesAspectValuesResponse(); response.setEntityName(entityName); @@ -235,56 +235,87 @@ public Task ingestProposal( @ActionParam(PARAM_PROPOSAL) @Nonnull MetadataChangeProposal metadataChangeProposal, @ActionParam(PARAM_ASYNC) @Optional(UNSET) String async) throws URISyntaxException { - log.info("INGEST PROPOSAL proposal: {}", metadataChangeProposal); + log.info("INGEST PROPOSAL proposal: {}", metadataChangeProposal); + + final boolean asyncBool; + if (UNSET.equals(async)) { + asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME)); + } else { + asyncBool = Boolean.parseBoolean(async); + } + + return ingestProposals(List.of(metadataChangeProposal), asyncBool); + } + + @Action(name = ACTION_INGEST_PROPOSAL_BATCH) + @Nonnull + @WithSpan + public Task ingestProposalBatch( + @ActionParam(PARAM_PROPOSALS) @Nonnull MetadataChangeProposal[] metadataChangeProposals, + @ActionParam(PARAM_ASYNC) @Optional(UNSET) String async) + throws URISyntaxException { + log.info("INGEST PROPOSAL BATCH proposals: {}", Arrays.asList(metadataChangeProposals)); + + final boolean asyncBool; + if (UNSET.equals(async)) { + asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME)); + } else { + asyncBool = Boolean.parseBoolean(async); + } + + return ingestProposals(Arrays.asList(metadataChangeProposals), asyncBool); + } - final boolean asyncBool; - if (UNSET.equals(async)) { - asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME)); - } else { - asyncBool = Boolean.parseBoolean(async); - } + private Task ingestProposals( + @Nonnull List metadataChangeProposals, + boolean asyncBool) + throws URISyntaxException { Authentication authentication = AuthenticationContext.getAuthentication(); - final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_INGEST_PROPOSAL, metadataChangeProposal.getEntityType()), _authorizer, authentication, true); - - /* - Ingest Authorization Checks - */ - List> exceptions = isAPIAuthorized(authentication, _authorizer, ENTITY, - opContext.getEntityRegistry(), List.of(metadataChangeProposal)) + + Set entityTypes = metadataChangeProposals.stream() + .map(MetadataChangeProposal::getEntityType) + .collect(Collectors.toSet()); + final OperationContext opContext = OperationContext.asSession( + systemOperationContext, RequestContext.builder().buildRestli(authentication.getActor().toUrnStr(), getContext(), ACTION_INGEST_PROPOSAL, entityTypes), _authorizer, authentication, true); + + // Ingest Authorization Checks + List> exceptions = isAPIAuthorized(authentication, _authorizer, ENTITY, + opContext.getEntityRegistry(), metadataChangeProposals) .stream().filter(p -> p.getSecond() != HttpStatus.S_200_OK.getCode()) .collect(Collectors.toList()); - if (!exceptions.isEmpty()) { - throw new RestLiServiceException( - HttpStatus.S_403_FORBIDDEN, "User is unauthorized to modify entity: " + exceptions.stream() + if (!exceptions.isEmpty()) { + String errorMessages = exceptions.stream() .map(ex -> String.format("HttpStatus: %s Urn: %s", ex.getSecond(), ex.getFirst().getEntityUrn())) - .collect(Collectors.toList())); - } - + .collect(Collectors.joining(", ")); + throw new RestLiServiceException( + HttpStatus.S_403_FORBIDDEN, "User is unauthorized to modify entity: " + errorMessages); + } String actorUrnStr = authentication.getActor().toUrnStr(); final AuditStamp auditStamp = new AuditStamp().setTime(_clock.millis()).setActor(Urn.createFromString(actorUrnStr)); return RestliUtil.toTask(() -> { - log.debug("Proposal: {}", metadataChangeProposal); + log.debug("Proposals: {}", metadataChangeProposals); try { final AspectsBatch batch = AspectsBatchImpl.builder() - .mcps(List.of(metadataChangeProposal), auditStamp, opContext.getRetrieverContext().get()) + .mcps(metadataChangeProposals, auditStamp, opContext.getRetrieverContext().get()) .build(); Set results = _entityService.ingestProposal(opContext, batch, asyncBool); - java.util.Optional one = results.stream().findFirst(); + for (IngestResult result : results) { + // Update runIds, only works for existing documents, so ES document must exist + Urn resultUrn = result.getUrn(); - // Update runIds, only works for existing documents, so ES document must exist - Urn resultUrn = one.map(IngestResult::getUrn).orElse(metadataChangeProposal.getEntityUrn()); - if (one.map(result -> result.isProcessedMCL() || result.isUpdate()).orElse(false)) { - tryIndexRunId(opContext, - resultUrn, metadataChangeProposal.getSystemMetadata(), entitySearchService); + if (resultUrn != null && (result.isProcessedMCL() || result.isUpdate())) { + tryIndexRunId(opContext, resultUrn, result.getRequest().getSystemMetadata(), entitySearchService); + } } - return resultUrn.toString(); + + // TODO: We don't actually use this return value anywhere. Maybe we should just stop returning it altogether? + return "success"; } catch (ValidationException e) { throw new RestLiServiceException(HttpStatus.S_422_UNPROCESSABLE_ENTITY, e.getMessage()); } @@ -310,7 +341,7 @@ public Task getCount( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to get aspect counts."); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_GET_COUNT, List.of()), _authorizer, authentication, true); + systemOperationContext, RequestContext.builder().buildRestli(authentication.getActor().toUrnStr(), getContext(), ACTION_GET_COUNT, List.of()), _authorizer, authentication, true); return _entityService.getCountAspect(opContext, aspectName, urnLike); }, @@ -339,7 +370,7 @@ public Task restoreIndices( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to update entities."); } - return Utils.restoreIndices(systemOperationContext, + return Utils.restoreIndices(systemOperationContext, getContext(), aspectName, urn, urnLike, start, batchSize, limit, gePitEpochMs, lePitEpochMs, _authorizer, _entityService); }, MetricRegistry.name(this.getClass(), "restoreIndices")); diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java index 970e18cb7bee0..599bbf9ce4df6 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java @@ -95,7 +95,7 @@ public Task rollback( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to update entity"); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli("rollback", List.of()), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), "rollback", List.of()), authorizer, auth, true); log.info("ROLLBACK RUN runId: {} dry run: {}", runId, dryRun); @@ -171,7 +171,7 @@ public Task describe( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to get entity"); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli("describe", List.of()), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), "describe", List.of()), authorizer, auth, true); List summaries = systemMetadataService.findByRunId( diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java index 4116b8ad30b94..e79dda3425682 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java @@ -200,7 +200,7 @@ public Task get( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to get entity " + urn); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli("restrictedService", urn.getEntityType()), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), "restrictedService", urn.getEntityType()), authorizer, auth, true); return RestliUtil.toTask( () -> { @@ -208,7 +208,7 @@ public Task get( aspectNames == null ? Collections.emptySet() : new HashSet<>(Arrays.asList(aspectNames)); - final Entity entity = entityService.getEntity(opContext, urn, projectedAspects); + final Entity entity = entityService.getEntity(opContext, urn, projectedAspects, true); if (entity == null) { throw RestliUtil.resourceNotFoundException(String.format("Did not find %s", urnStr)); } @@ -240,7 +240,7 @@ public Task> batchGet( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to get entities: " + urnStrs); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli("batchGet", urnStrs), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), "batchGet", urnStrs), authorizer, auth, true); return RestliUtil.toTask( () -> { @@ -248,7 +248,7 @@ public Task> batchGet( aspectNames == null ? Collections.emptySet() : new HashSet<>(Arrays.asList(aspectNames)); - return entityService.getEntities(opContext, urns, projectedAspects).entrySet().stream() + return entityService.getEntities(opContext, urns, projectedAspects, true).entrySet().stream() .collect( Collectors.toMap( entry -> entry.getKey().toString(), @@ -289,7 +289,7 @@ public Task ingest( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to edit entity " + urn); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_INGEST, urn.getEntityType()), authorizer, authentication, true); + systemOperationContext, RequestContext.builder().buildRestli(authentication.getActor().toUrnStr(), getContext(), ACTION_INGEST, urn.getEntityType()), authorizer, authentication, true); try { validateOrThrow(entity); @@ -334,7 +334,7 @@ public Task batchIngest( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to edit entities."); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_BATCH_INGEST, urns.stream() + systemOperationContext, RequestContext.builder().buildRestli(authentication.getActor().toUrnStr(), getContext(), ACTION_BATCH_INGEST, urns.stream() .map(Urn::getEntityType).collect(Collectors.toList())), authorizer, authentication, true); for (Entity entity : entities) { @@ -394,7 +394,7 @@ public Task search( } OperationContext opContext = OperationContext.asSession(systemOperationContext, - RequestContext.builder().buildRestli(ACTION_SEARCH, entityName), authorizer, auth, true) + RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_SEARCH, entityName), authorizer, auth, true) .withSearchFlags(flags -> searchFlags != null ? searchFlags : new SearchFlags().setFulltext(Boolean.TRUE.equals(fulltext))); log.info("GET SEARCH RESULTS for {} with query {}", entityName, input); @@ -434,7 +434,7 @@ public Task searchAcrossEntities( final Authentication auth = AuthenticationContext.getAuthentication(); OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_SEARCH_ACROSS_ENTITIES, entities), authorizer, auth, true) + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_SEARCH_ACROSS_ENTITIES, entities), authorizer, auth, true) .withSearchFlags(flags -> searchFlags != null ? searchFlags : new SearchFlags().setFulltext(true)); List entityList = searchService.getEntitiesToSearch(opContext, entities == null ? Collections.emptyList() : Arrays.asList(entities), count); @@ -478,7 +478,7 @@ public Task scrollAcrossEntities( final Authentication auth = AuthenticationContext.getAuthentication(); OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_SCROLL_ACROSS_ENTITIES, entities), authorizer, auth, true) + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_SCROLL_ACROSS_ENTITIES, entities), authorizer, auth, true) .withSearchFlags(flags -> searchFlags != null ? searchFlags : new SearchFlags().setFulltext(true)); List entityList = searchService.getEntitiesToSearch(opContext, entities == null ? Collections.emptyList() : Arrays.asList(entities), count); @@ -547,7 +547,7 @@ public Task searchAcrossLineage( } OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_SEARCH_ACROSS_LINEAGE, entities), authorizer, auth, true) + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_SEARCH_ACROSS_LINEAGE, entities), authorizer, auth, true) .withSearchFlags(flags -> (searchFlags != null ? searchFlags : new SearchFlags().setFulltext(true)) .setIncludeRestricted(true)) .withLineageFlags(flags -> flags.setStartTimeMillis(startTimeMillis, SetMode.REMOVE_IF_NULL) @@ -606,7 +606,7 @@ public Task scrollAcrossLineage( } OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_SCROLL_ACROSS_LINEAGE, entities), + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_SCROLL_ACROSS_LINEAGE, entities), authorizer, auth, true) .withSearchFlags(flags -> (searchFlags != null ? searchFlags : new SearchFlags().setSkipCache(true)) .setIncludeRestricted(true)) @@ -661,7 +661,7 @@ public Task list( } OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_LIST, entityName), authorizer, auth, true) + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_LIST, entityName), authorizer, auth, true) .withSearchFlags(flags -> new SearchFlags().setFulltext(false)); log.info("GET LIST RESULTS for {} with filter {}", entityName, filter); @@ -702,7 +702,7 @@ public Task autocomplete( } OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_AUTOCOMPLETE, entityName), authorizer, auth, true) + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_AUTOCOMPLETE, entityName), authorizer, auth, true) .withSearchFlags(flags -> searchFlags != null ? searchFlags : flags); return RestliUtil.toTask( @@ -740,7 +740,7 @@ public Task browse( } OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_BROWSE, entityName), authorizer, auth, true) + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_BROWSE, entityName), authorizer, auth, true) .withSearchFlags(flags -> searchFlags != null ? searchFlags : flags); log.info("GET BROWSE RESULTS for {} at path {}", entityName, path); @@ -779,7 +779,7 @@ public Task getBrowsePaths( log.info("GET BROWSE PATHS for {}", urn); OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_GET_BROWSE_PATHS, urn.getEntityType()), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_GET_BROWSE_PATHS, urn.getEntityType()), authorizer, auth, true); return RestliUtil.toTask( () -> new StringArray(entitySearchService.getBrowsePaths(opContext, urnToEntityName(urn), urn)), @@ -848,7 +848,7 @@ public Task deleteEntities( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to delete entities."); } OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli("deleteAll", urns), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), "deleteAll", urns), authorizer, auth, true); response.setEntitiesAffected(urns.size()); response.setEntitiesDeleted( @@ -899,7 +899,7 @@ public Task deleteEntity( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to delete entity: " + urnStr); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_DELETE, urn.getEntityType()), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_DELETE, urn.getEntityType()), authorizer, auth, true); return RestliUtil.toTask( () -> { @@ -961,7 +961,7 @@ private Long deleteTimeseriesAspects( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to delete entity " + urn); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli("deleteTimeseriesAspects", urn.getEntityType()), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), "deleteTimeseriesAspects", urn.getEntityType()), authorizer, auth, true); // Construct the filter. List criteria = new ArrayList<>(); @@ -1017,7 +1017,7 @@ public Task deleteReferencesTo( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to delete entity " + urnStr); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli("deleteReferences", urn.getEntityType()), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), "deleteReferences", urn.getEntityType()), authorizer, auth, true); return RestliUtil.toTask( () -> deleteEntityService.deleteReferencesTo(opContext, urn, dryRun), @@ -1062,7 +1062,7 @@ public Task getTotalEntityCount(@ActionParam(PARAM_ENTITY) @Nonnull String HttpStatus.S_403_FORBIDDEN, "User is unauthorized to get entity counts."); } OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli("getTotalEntityCount", entityName), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), "getTotalEntityCount", entityName), authorizer, auth, true); return RestliUtil.toTask(() -> entitySearchService.docCount(opContext, entityName)); } @@ -1080,7 +1080,7 @@ public Task batchGetTotalEntityCount( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to get entity counts."); } OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli("batchGetTotalEntityCount", entityNames), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), "batchGetTotalEntityCount", entityNames), authorizer, auth, true); return RestliUtil.toTask( () -> new LongMap(searchService.docCountPerEntity(opContext, Arrays.asList(entityNames)))); } @@ -1103,7 +1103,7 @@ public Task listUrns( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to search."); } OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_LIST_URNS, entityName), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_LIST_URNS, entityName), authorizer, auth, true); log.info("LIST URNS for {} with start {} and count {}", entityName, start, count); return RestliUtil.toTask(() -> { @@ -1145,7 +1145,7 @@ public Task applyRetention( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to apply retention."); } OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_APPLY_RETENTION, resourceSpec.getType()), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_APPLY_RETENTION, resourceSpec.getType()), authorizer, auth, true); return RestliUtil.toTask( () -> entityService.batchApplyRetention(opContext, start, count, attemptWithVersion, aspectName, urn), @@ -1171,7 +1171,7 @@ public Task filter( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to search."); } OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_FILTER, entityName), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_FILTER, entityName), authorizer, auth, true); log.info("FILTER RESULTS for {} with filter {}", entityName, filter); return RestliUtil.toTask( () -> { @@ -1206,7 +1206,7 @@ public Task exists(@ActionParam(PARAM_URN) @Nonnull String urnStr, @Act HttpStatus.S_403_FORBIDDEN, "User is unauthorized check entity existence: " + urnStr); } OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_EXISTS, urn.getEntityType()), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_EXISTS, urn.getEntityType()), authorizer, auth, true); log.info("EXISTS for {}", urnStr); final boolean includeRemoved = includeSoftDelete == null || includeSoftDelete; return RestliUtil.toTask( diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java index af74040e774e5..9052f0240266a 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java @@ -79,7 +79,7 @@ public Task get( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to get entity " + urn); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli("getEntityV2", urn.getEntityType()), _authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), "getEntityV2", urn.getEntityType()), _authorizer, auth, true); return RestliUtil.toTask( () -> { @@ -123,7 +123,7 @@ public Task> batchGet( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to get entities " + urnStrs); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli("getEntityV2", urns.stream().map(Urn::getEntityType).collect(Collectors.toList())), _authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), "getEntityV2", urns.stream().map(Urn::getEntityType).collect(Collectors.toList())), _authorizer, auth, true); if (urns.size() <= 0) { return Task.value(Collections.emptyMap()); diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java index 794115ec17d1e..d6c91ba7dcaa3 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityVersionedV2Resource.java @@ -88,7 +88,7 @@ public Task> batchGetVersioned( "User is unauthorized to get entities " + versionedUrnStrs); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli("authorizerChain", urns.stream() + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), "authorizerChain", urns.stream() .map(Urn::getEntityType).collect(Collectors.toList())), _authorizer, auth, true); log.debug("BATCH GET VERSIONED V2 {}", versionedUrnStrs); diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/OperationsResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/OperationsResource.java index c92f7207aa655..42d0bf11c505d 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/OperationsResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/OperationsResource.java @@ -107,7 +107,7 @@ public Task restoreIndices( @ActionParam("gePitEpochMs") @Optional @Nullable Long gePitEpochMs, @ActionParam("lePitEpochMs") @Optional @Nullable Long lePitEpochMs) { return RestliUtil.toTask( - () -> Utils.restoreIndices(systemOperationContext, + () -> Utils.restoreIndices(systemOperationContext, getContext(), aspectName, urn, urnLike, start, batchSize, limit, gePitEpochMs, lePitEpochMs, _authorizer, _entityService), MetricRegistry.name(this.getClass(), "restoreIndices")); } @@ -202,7 +202,7 @@ public Task getIndexSizes() { HttpStatus.S_403_FORBIDDEN, "User is unauthorized to get index sizes."); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_GET_INDEX_SIZES, List.of()), _authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_GET_INDEX_SIZES, List.of()), _authorizer, auth, true); TimeseriesIndicesSizesResult result = new TimeseriesIndicesSizesResult(); result.setIndexSizes( @@ -232,7 +232,7 @@ String executeTruncateTimeseriesAspect( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to truncate timeseries index"); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli("executeTruncateTimeseriesAspect", entityType), _authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), "executeTruncateTimeseriesAspect", entityType), _authorizer, auth, true); if (forceDeleteByQuery != null && forceDeleteByQuery.equals(forceReindex)) { return "please only set forceReindex OR forceDeleteByQuery flags"; diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/Utils.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/Utils.java index 2c411f9ad960e..54c1029edcab0 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/Utils.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/Utils.java @@ -13,6 +13,7 @@ import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesResult; import com.linkedin.restli.common.HttpStatus; +import com.linkedin.restli.server.ResourceContext; import com.linkedin.restli.server.RestLiServiceException; import java.util.HashMap; import java.util.List; @@ -32,7 +33,8 @@ public class Utils { private Utils() {} public static String restoreIndices( - @Nonnull OperationContext systemOperationContext, + @Nonnull OperationContext systemOperationContext, + @Nonnull ResourceContext resourceContext, @Nonnull String aspectName, @Nullable String urn, @Nullable String urnLike, @@ -59,7 +61,7 @@ public static String restoreIndices( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to restore indices."); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli("restoreIndices", List.of()), authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), resourceContext, "restoreIndices", List.of()), authorizer, auth, true); RestoreIndicesArgs args = new RestoreIndicesArgs() diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java index 82e0e000a31be..518dfecd57680 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java @@ -141,7 +141,7 @@ public Task batchIngest(@ActionParam(PARAM_BUCKETS) @Nonnull UsageAggregat HttpStatus.S_403_FORBIDDEN, "User is unauthorized to edit entities."); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_BATCH_INGEST, urns.stream() + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_BATCH_INGEST, urns.stream() .map(Urn::getEntityType).collect(Collectors.toList())), _authorizer, auth, true); for (UsageAggregation agg : buckets) { @@ -180,7 +180,7 @@ public Task query( HttpStatus.S_403_FORBIDDEN, "User is unauthorized to query usage."); } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_QUERY, resourceUrn.getEntityType()), _authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_QUERY, resourceUrn.getEntityType()), _authorizer, auth, true); return UsageServiceUtil.query(opContext, _timeseriesAspectService, resource, duration, startTime, endTime, maxBuckets); }, @@ -207,7 +207,7 @@ public Task queryRange( } final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_QUERY_RANGE, resourceUrn.getEntityType()), _authorizer, auth, true); + systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), ACTION_QUERY_RANGE, resourceUrn.getEntityType()), _authorizer, auth, true); return RestliUtil.toTask( () -> UsageServiceUtil.queryRange(opContext, _timeseriesAspectService, resource, duration, range), MetricRegistry.name(this.getClass(), "queryRange")); diff --git a/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java b/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java index ed0e8c0858526..9872f45648d7b 100644 --- a/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java +++ b/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java @@ -97,7 +97,7 @@ public void testAsyncDefaultAspects() throws URISyntaxException { .recordTemplate(mcp.getAspect()) .auditStamp(new AuditStamp()) .metadataChangeProposal(mcp) - .build(opContext.getRetrieverContext().get().getAspectRetriever()); + .build(opContext.getAspectRetrieverOpt().get()); when(aspectDao.runInTransactionWithRetry(any(), any(), anyInt())) .thenReturn( List.of(List.of( diff --git a/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java b/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java index d220883d24132..aaf90d279e0bd 100644 --- a/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java +++ b/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java @@ -40,9 +40,6 @@ public MockTimeseriesAspectService(long count, long filteredCount, String taskId this._taskId = taskId; } - @Override - public void configure() {} - @Override public long countByFilter( @Nonnull OperationContext operationContext, diff --git a/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/SchemaRegistryController.java b/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/SchemaRegistryController.java index b9b4dd1ec01b1..d73b353f38ae7 100644 --- a/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/SchemaRegistryController.java +++ b/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/SchemaRegistryController.java @@ -36,6 +36,7 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; /** DataHub Rest Controller implementation for Confluent's Schema Registry OpenAPI spec. */ @@ -249,14 +250,21 @@ public ResponseEntity deleteTopLevelConfig() { @Override public ResponseEntity getSubjectLevelConfig(String subject, Boolean defaultToGlobal) { - log.error("[ConfigApi] getSubjectLevelConfig method not implemented"); - return ConfigApi.super.getSubjectLevelConfig(subject, defaultToGlobal); + return getTopLevelConfig(); } + @RequestMapping( + value = {"/config", "/config/"}, + produces = { + "application/vnd.schemaregistry.v1+json", + "application/vnd.schemaregistry+json; qs=0.9", + "application/json; qs=0.5" + }, + method = RequestMethod.GET) @Override public ResponseEntity getTopLevelConfig() { - log.error("[ConfigApi] getTopLevelConfig method not implemented"); - return ConfigApi.super.getTopLevelConfig(); + return ResponseEntity.ok( + new Config().compatibilityLevel(Config.CompatibilityLevelEnum.BACKWARD)); } @Override diff --git a/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTestConfiguration.java b/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTestConfiguration.java index 7ab673b0a46fe..6901cd665f166 100644 --- a/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTestConfiguration.java +++ b/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTestConfiguration.java @@ -1,6 +1,7 @@ package io.datahubproject.openapi.test; import com.linkedin.metadata.dao.producer.KafkaHealthChecker; +import com.linkedin.metadata.models.registry.EntityRegistry; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.context.annotation.ComponentScan; @@ -11,4 +12,6 @@ @ComponentScan(basePackages = {"com.linkedin.gms.factory.kafka", "com.linkedin.gms.factory.config"}) public class SchemaRegistryControllerTestConfiguration { @MockBean KafkaHealthChecker kafkaHealthChecker; + + @MockBean EntityRegistry entityRegistry; } diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java index ab8786aeca8d6..aed9b97411ff6 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java @@ -6,6 +6,11 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.linkedin.common.AuditStamp; +import com.linkedin.common.FormAssociation; +import com.linkedin.common.FormAssociationArray; +import com.linkedin.common.FormVerificationAssociation; +import com.linkedin.common.FormVerificationAssociationArray; +import com.linkedin.common.Forms; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.schema.PathSpec; @@ -22,14 +27,24 @@ import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.RelationshipFieldSpec; import com.linkedin.metadata.models.extractor.FieldExtractor; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.run.DeleteReferencesResponse; import com.linkedin.metadata.run.RelatedAspect; import com.linkedin.metadata.run.RelatedAspectArray; +import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.search.ScrollResult; +import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import io.datahubproject.metadata.context.OperationContext; import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -39,6 +54,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import lombok.AllArgsConstructor; import lombok.Data; import lombok.RequiredArgsConstructor; @@ -50,8 +66,10 @@ public class DeleteEntityService { private final EntityService _entityService; private final GraphService _graphService; + private final EntitySearchService _searchService; private static final Integer ELASTIC_BATCH_DELETE_SLEEP_SEC = 5; + private static final Integer BATCH_SIZE = 1000; /** * Public endpoint that deletes references to a given urn across DataHub's metadata graph. This is @@ -65,7 +83,14 @@ public class DeleteEntityService { */ public DeleteReferencesResponse deleteReferencesTo( @Nonnull OperationContext opContext, final Urn urn, final boolean dryRun) { + // TODO: update DeleteReferencesResponse to have searchAspects and provide more helpful comment + // in CLI final DeleteReferencesResponse result = new DeleteReferencesResponse(); + + // Delete references for entities referencing the deleted urn with searchables. + // Only works for Form deletion for now + int totalSearchAssetCount = deleteSearchReferences(opContext, urn, dryRun); + RelatedEntitiesResult relatedEntities = _graphService.findRelatedEntities( null, @@ -90,7 +115,7 @@ public DeleteReferencesResponse deleteReferencesTo( .collect(Collectors.toList()); result.setRelatedAspects(new RelatedAspectArray(relatedAspects)); - result.setTotal(relatedEntities.getTotal()); + result.setTotal(relatedEntities.getTotal() + totalSearchAssetCount); if (dryRun) { return result; @@ -489,6 +514,244 @@ private void handleError(final DeleteEntityServiceError error) { // NO-OP for now. } + private static class AssetScrollResult { + String scrollId; + List assets; + int totalAssetCount; + } + + /** + * Method that will find references through the search index and clean aspects up. Right now this + * only works for deleting Forms entities. Later, we need to extend this for all searchables. + */ + private int deleteSearchReferences( + @Nonnull OperationContext opContext, @Nonnull final Urn deletedUrn, final boolean dryRun) { + int totalAssetCount = 0; + String scrollId = null; + do { + AssetScrollResult assetScrollResult = + getAssetsReferencingUrn(opContext, deletedUrn, scrollId, dryRun); + List assetsReferencingUrn = assetScrollResult.assets; + totalAssetCount += assetScrollResult.totalAssetCount; + // if it's a dry run, exit early and stop looping over assets + scrollId = dryRun ? null : assetScrollResult.scrollId; + if (!dryRun) { + assetsReferencingUrn.forEach( + assetUrn -> { + List mcps = + deleteSearchReferencesForAsset(opContext, assetUrn, deletedUrn); + mcps.forEach( + mcp -> _entityService.ingestProposal(opContext, mcp, createAuditStamp(), true)); + }); + } + } while (scrollId != null); + return totalAssetCount; + } + + /** + * Get all of the asset urns that reference the deleted urn This is hardcoded for forms right now, + * we will use a more generic field to work with all searchables later. + * + *

TODO: Use the new referencedUrns index field once that exists + */ + private AssetScrollResult getAssetsReferencingUrn( + @Nonnull OperationContext opContext, + @Nonnull final Urn deletedUrn, + @Nullable String scrollId, + final boolean dryRun) { + AssetScrollResult result = new AssetScrollResult(); + result.scrollId = null; + result.totalAssetCount = 0; + result.assets = new ArrayList<>(); + + if (deletedUrn.getEntityType().equals("form")) { + // first, get all entities with this form assigned on it + final CriterionArray incompleteFormsArray = new CriterionArray(); + incompleteFormsArray.add( + new Criterion() + .setField("incompleteForms") + .setValue(deletedUrn.toString()) + .setCondition(Condition.EQUAL)); + final CriterionArray completedFormsArray = new CriterionArray(); + completedFormsArray.add( + new Criterion() + .setField("completedForms") + .setValue(deletedUrn.toString()) + .setCondition(Condition.EQUAL)); + // next, get all metadata tests created for this form + final CriterionArray metadataTestSourceArray = new CriterionArray(); + metadataTestSourceArray.add( + new Criterion() + .setField("sourceEntity") + .setValue(deletedUrn.toString()) + .setCondition(Condition.EQUAL)); + metadataTestSourceArray.add( + new Criterion().setField("sourceType").setValue("FORMS").setCondition(Condition.EQUAL)); + Filter filter = + new Filter() + .setOr( + new ConjunctiveCriterionArray( + new ConjunctiveCriterion().setAnd(incompleteFormsArray), + new ConjunctiveCriterion().setAnd(completedFormsArray), + new ConjunctiveCriterion().setAnd(metadataTestSourceArray))); + ScrollResult scrollResult = + _searchService.structuredScroll( + opContext, + ImmutableList.of( + "dataset", + "dataJob", + "dataFlow", + "chart", + "dashboard", + "corpuser", + "corpGroup", + "domain", + "container", + "glossaryTerm", + "glossaryNode", + "mlModel", + "mlModelGroup", + "mlFeatureTable", + "mlFeature", + "mlPrimaryKey", + "schemaField", + "dataProduct", + "test"), + "*", + filter, + null, + scrollId, + "5m", + dryRun ? 1 : BATCH_SIZE); // need to pass in 1 for count otherwise get index error + if (scrollResult.getNumEntities() == 0 || scrollResult.getEntities().size() == 0) { + return result; + } + result.scrollId = scrollResult.getScrollId(); + result.totalAssetCount = scrollResult.getNumEntities(); + result.assets = + scrollResult.getEntities().stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toList()); + return result; + } + return result; + } + + /** Get all the aspects that need updating, then loop over and update them */ + private List deleteSearchReferencesForAsset( + @Nonnull OperationContext opContext, + @Nonnull final Urn assetUrn, + @Nonnull final Urn deletedUrn) { + // delete entities that should be deleted first + if (shouldDeleteAssetReferencingUrn(assetUrn, deletedUrn)) { + _entityService.deleteUrn(opContext, assetUrn); + } + + List mcps = new ArrayList<>(); + List aspectsToUpdate = getAspectsToUpdate(deletedUrn); + aspectsToUpdate.forEach( + aspectName -> { + try { + MetadataChangeProposal mcp = + updateAspectForSearchReference(opContext, assetUrn, deletedUrn, aspectName); + if (mcp != null) { + mcps.add(mcp); + } + } catch (Exception e) { + log.error( + String.format( + "Error trying to update aspect %s for asset %s when deleting %s", + aspectName, assetUrn, deletedUrn), + e); + } + }); + return mcps; + } + + /** + * Get the names of all the aspects that need to be updated when the deleted urn is removed. + * + *

TODO: extend this to support other types of deletes and be more dynamic depending on aspects + * that the asset has + */ + private List getAspectsToUpdate(@Nonnull final Urn deletedUrn) { + if (deletedUrn.getEntityType().equals("form")) { + return ImmutableList.of("forms"); + } + return new ArrayList<>(); + } + + /** + * Determine whether the asset referencing the deleted urn should be deleted itself. + * + *

TODO: extend this to support other types of deletes and be more dynamic depending on aspects + * that the asset has + */ + private boolean shouldDeleteAssetReferencingUrn( + @Nonnull final Urn assetUrn, @Nonnull final Urn deletedUrn) { + if (assetUrn.getEntityType().equals("test") && deletedUrn.getEntityType().equals("form")) { + return true; + } + return false; + } + + @Nullable + private MetadataChangeProposal updateAspectForSearchReference( + @Nonnull OperationContext opContext, + @Nonnull final Urn assetUrn, + @Nonnull final Urn deletedUrn, + @Nonnull final String aspectName) { + if (aspectName.equals("forms")) { + return updateFormsAspect(opContext, assetUrn, deletedUrn); + } + return null; + } + + @Nullable + private MetadataChangeProposal updateFormsAspect( + @Nonnull OperationContext opContext, + @Nonnull final Urn assetUrn, + @Nonnull final Urn deletedUrn) { + RecordTemplate record = _entityService.getLatestAspect(opContext, assetUrn, "forms"); + if (record == null) return null; + + Forms formsAspect = new Forms(record.data()); + final AtomicReference updatedAspect; + try { + updatedAspect = new AtomicReference<>(formsAspect.copy()); + } catch (Exception e) { + throw new RuntimeException("Failed to copy the forms aspect for updating", e); + } + + List incompleteForms = + formsAspect.getIncompleteForms().stream() + .filter(incompleteForm -> !incompleteForm.getUrn().equals(deletedUrn)) + .collect(Collectors.toList()); + List completedForms = + formsAspect.getCompletedForms().stream() + .filter(completedForm -> completedForm.getUrn() != deletedUrn) + .collect(Collectors.toList()); + final List verifications = + formsAspect.getVerifications().stream() + .filter(verification -> verification.getForm() != deletedUrn) + .collect(Collectors.toList()); + + updatedAspect.get().setIncompleteForms(new FormAssociationArray(incompleteForms)); + updatedAspect.get().setCompletedForms(new FormAssociationArray(completedForms)); + updatedAspect.get().setVerifications(new FormVerificationAssociationArray(verifications)); + + if (!formsAspect.equals(updatedAspect.get())) { + return AspectUtils.buildMetadataChangeProposal(assetUrn, "forms", updatedAspect.get()); + } + return null; + } + + private AuditStamp createAuditStamp() { + return new AuditStamp() + .setActor(UrnUtils.getUrn(Constants.SYSTEM_ACTOR)) + .setTime(System.currentTimeMillis()); + } + @AllArgsConstructor @Data private static class DeleteEntityServiceError { diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java index 0794ba72ff692..283cff5d2e19d 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -115,12 +115,23 @@ default boolean exists(@Nonnull OperationContext opContext, @Nonnull Urn urn) { * * @param urns set of urns to fetch aspects for * @param aspectNames aspects to fetch for each urn in urns set + * @param alwaysIncludeKeyAspect historically the key aspect was always added, allow disabling + * this behavior * @return a map of provided {@link Urn} to a List containing the requested aspects. */ Map> getLatestAspects( @Nonnull OperationContext opContext, @Nonnull final Set urns, - @Nonnull final Set aspectNames); + @Nonnull final Set aspectNames, + boolean alwaysIncludeKeyAspect); + + @Deprecated + default Map> getLatestAspects( + @Nonnull OperationContext opContext, + @Nonnull final Set urns, + @Nonnull final Set aspectNames) { + return getLatestAspects(opContext, urns, aspectNames, true); + } Map getLatestAspectsForUrn( @Nonnull OperationContext opContext, @@ -152,15 +163,28 @@ RecordTemplate getAspect( * @param entityName name of the entity to fetch * @param urn urn of entity to fetch * @param aspectNames set of aspects to fetch + * @param alwaysIncludeKeyAspect historically the key aspect was always added, allow disabling + * this behavior * @return a map of {@link Urn} to {@link Entity} object */ EntityResponse getEntityV2( @Nonnull OperationContext opContext, @Nonnull final String entityName, @Nonnull final Urn urn, - @Nonnull final Set aspectNames) + @Nonnull final Set aspectNames, + boolean alwaysIncludeKeyAspect) throws URISyntaxException; + @Deprecated + default EntityResponse getEntityV2( + @Nonnull OperationContext opContext, + @Nonnull final String entityName, + @Nonnull final Urn urn, + @Nonnull final Set aspectNames) + throws URISyntaxException { + return getEntityV2(opContext, entityName, urn, aspectNames, true); + } + /** * Retrieves the latest aspects for the given set of urns as dynamic aspect objects (Without * having to define union objects) @@ -168,15 +192,27 @@ EntityResponse getEntityV2( * @param entityName name of the entity to fetch * @param urns set of urns to fetch * @param aspectNames set of aspects to fetch + * @param alwaysIncludeKeyAspect historically the key aspect was always added, allow disabling + * this behavior * @return a map of {@link Urn} to {@link Entity} object */ Map getEntitiesV2( @Nonnull OperationContext opContext, @Nonnull final String entityName, @Nonnull final Set urns, - @Nonnull final Set aspectNames) + @Nonnull final Set aspectNames, + boolean alwaysIncludeKeyAspect) throws URISyntaxException; + default Map getEntitiesV2( + @Nonnull OperationContext opContext, + @Nonnull final String entityName, + @Nonnull final Set urns, + @Nonnull final Set aspectNames) + throws URISyntaxException { + return getEntitiesV2(opContext, entityName, urns, aspectNames, true); + } + /** * Retrieves the aspects for the given set of urns and versions as dynamic aspect objects (Without * having to define union objects) @@ -184,39 +220,90 @@ Map getEntitiesV2( * @param versionedUrns set of urns to fetch with versions of aspects specified in a specialized * string * @param aspectNames set of aspects to fetch + * @param alwaysIncludeKeyAspect historically the key aspect was always added, allow disabling + * this behavior * @return a map of {@link Urn} to {@link Entity} object */ Map getEntitiesVersionedV2( @Nonnull OperationContext opContext, @Nonnull final Set versionedUrns, - @Nonnull final Set aspectNames) + @Nonnull final Set aspectNames, + boolean alwaysIncludeKeyAspect) throws URISyntaxException; + @Deprecated + default Map getEntitiesVersionedV2( + @Nonnull OperationContext opContext, + @Nonnull final Set versionedUrns, + @Nonnull final Set aspectNames) + throws URISyntaxException { + return getEntitiesVersionedV2(opContext, versionedUrns, aspectNames, true); + } + /** * Retrieves the latest aspects for the given set of urns as a list of enveloped aspects * * @param urns set of urns to fetch * @param aspectNames set of aspects to fetch + * @param alwaysIncludeKeyAspect historically the key aspect was always added, allow disabling + * this behavior * @return a map of {@link Urn} to {@link EnvelopedAspect} object */ Map> getLatestEnvelopedAspects( - @Nonnull OperationContext opContext, @Nonnull Set urns, @Nonnull Set aspectNames) + @Nonnull OperationContext opContext, + @Nonnull Set urns, + @Nonnull Set aspectNames, + boolean alwaysIncludeKeyAspect) throws URISyntaxException; + /** + * Retrieve the specified aspect versions for the given URNs + * + * @param opContext operation context + * @param urnAspectVersions map of the urn's aspect versions + * @param alwaysIncludeKeyAspect whether to include the key aspect + * @return enveloped aspects with the specific version + * @throws URISyntaxException + */ + Map> getEnvelopedVersionedAspects( + @Nonnull OperationContext opContext, + @Nonnull Map> urnAspectVersions, + boolean alwaysIncludeKeyAspect) + throws URISyntaxException; + + @Deprecated + default Map> getLatestEnvelopedAspects( + @Nonnull OperationContext opContext, @Nonnull Set urns, @Nonnull Set aspectNames) + throws URISyntaxException { + return getLatestEnvelopedAspects(opContext, urns, aspectNames, true); + } + /** * Retrieves the latest aspects for the given set of urns as a list of enveloped aspects * * @param versionedUrns set of urns to fetch with versions of aspects specified in a specialized * string * @param aspectNames set of aspects to fetch + * @param alwaysIncludeKeyAspect historically the key aspect was always added, allow disabling + * this behavior * @return a map of {@link Urn} to {@link EnvelopedAspect} object */ Map> getVersionedEnvelopedAspects( @Nonnull OperationContext opContext, @Nonnull Set versionedUrns, - @Nonnull Set aspectNames) + @Nonnull Set aspectNames, + boolean alwaysIncludeKeyAspect) throws URISyntaxException; + @Deprecated + default Map> getVersionedEnvelopedAspects( + @Nonnull OperationContext opContext, + @Nonnull Set versionedUrns, + @Nonnull Set aspectNames) + throws URISyntaxException { + return getVersionedEnvelopedAspects(opContext, versionedUrns, aspectNames, true); + } + /** * Retrieves the latest aspect for the given urn as a list of enveloped aspects * @@ -320,13 +407,15 @@ ListUrnsResult listUrns( Entity getEntity( @Nonnull OperationContext opContext, @Nonnull final Urn urn, - @Nonnull final Set aspectNames); + @Nonnull final Set aspectNames, + boolean alwaysIncludeKeyAspect); @Deprecated Map getEntities( @Nonnull OperationContext opContext, @Nonnull final Set urns, - @Nonnull Set aspectNames); + @Nonnull Set aspectNames, + boolean alwaysIncludeKeyAspect); Pair, Boolean> alwaysProduceMCLAsync( @Nonnull OperationContext opContext, @@ -369,12 +458,18 @@ void ingestEntity( void setRetentionService(RetentionService retentionService); - RollbackResult deleteAspect( + default RollbackResult deleteAspect( @Nonnull OperationContext opContext, String urn, String aspectName, @Nonnull Map conditions, - boolean hardDelete); + boolean hardDelete) { + AspectRowSummary aspectRowSummary = + new AspectRowSummary().setUrn(urn).setAspectName(aspectName); + return rollbackWithConditions(opContext, List.of(aspectRowSummary), conditions, hardDelete) + .getRollbackResults() + .get(0); + } RollbackRunResult deleteUrn(@Nonnull OperationContext opContext, Urn urn); diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/RollbackResult.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/RollbackResult.java index 9955a58c65339..143d6102f2523 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/RollbackResult.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/RollbackResult.java @@ -1,8 +1,13 @@ package com.linkedin.metadata.entity; +import static com.linkedin.metadata.utils.PegasusUtils.constructMCL; +import static com.linkedin.metadata.utils.PegasusUtils.urnToEntityName; + +import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; import com.linkedin.events.metadata.ChangeType; +import com.linkedin.mxe.MetadataChangeLog; import com.linkedin.mxe.SystemMetadata; import lombok.Value; @@ -18,4 +23,22 @@ public class RollbackResult { public ChangeType changeType; public Boolean keyAffected; public Integer additionalRowsAffected; + + public boolean isNoOp() { + return oldValue == newValue; + } + + public MetadataChangeLog toMCL(AuditStamp auditStamp) { + return constructMCL( + null, + urnToEntityName(urn), + urn, + changeType, + aspectName, + auditStamp, + newValue, + newSystemMetadata, + oldValue, + oldSystemMetadata); + } } diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/RollbackRunResult.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/RollbackRunResult.java index a8c558df77e57..68ee66c2b50ed 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/RollbackRunResult.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/RollbackRunResult.java @@ -8,4 +8,5 @@ public class RollbackRunResult { public List rowsRolledBack; public Integer rowsDeletedFromEntityDeletion; + public List rollbackResults; } diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/UpdateAspectResult.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/UpdateAspectResult.java index e85e0567f963b..608a55674d859 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/UpdateAspectResult.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/UpdateAspectResult.java @@ -1,12 +1,19 @@ package com.linkedin.metadata.entity; +import static com.linkedin.metadata.utils.PegasusUtils.constructMCL; +import static com.linkedin.metadata.utils.PegasusUtils.urnToEntityName; + import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.batch.ChangeMCP; import com.linkedin.mxe.MetadataAuditOperation; +import com.linkedin.mxe.MetadataChangeLog; +import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.SystemMetadata; import java.util.concurrent.Future; +import javax.annotation.Nullable; import lombok.Builder; import lombok.Value; @@ -15,13 +22,35 @@ public class UpdateAspectResult { Urn urn; ChangeMCP request; - RecordTemplate oldValue; + @Nullable RecordTemplate oldValue; RecordTemplate newValue; - SystemMetadata oldSystemMetadata; + @Nullable SystemMetadata oldSystemMetadata; SystemMetadata newSystemMetadata; MetadataAuditOperation operation; AuditStamp auditStamp; long maxVersion; + @Nullable MetadataChangeProposal mcp; + /* + Whether the MCL was written to Elasticsearch prior to emitting the MCL + */ boolean processedMCL; Future mclFuture; + + public boolean isNoOp() { + return oldValue == newValue; + } + + public MetadataChangeLog toMCL() { + return constructMCL( + request.getMetadataChangeProposal(), + urnToEntityName(urn), + urn, + isNoOp() ? ChangeType.RESTATE : ChangeType.UPSERT, + request.getAspectName(), + auditStamp, + newValue, + newSystemMetadata, + oldValue, + oldSystemMetadata); + } } diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphService.java index ad8fdbbb06895..16305d6ec4bcf 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphService.java @@ -314,7 +314,7 @@ void removeEdgesFromNode( @Nonnull final List relationshipTypes, @Nonnull final RelationshipFilter relationshipFilter); - void configure(); + default void configure() {} /** Removes all edges and nodes from the graph. */ void clear(); diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java index a253f9ffc2531..a3db4b029b68b 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java @@ -15,7 +15,7 @@ public interface EntitySearchService { - void configure(); + default void configure() {} /** Clear all data within the service */ void clear(@Nonnull OperationContext opContext); diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/service/AssertionService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/AssertionService.java new file mode 100644 index 0000000000000..6b56d6b4dc295 --- /dev/null +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/service/AssertionService.java @@ -0,0 +1,181 @@ +package com.linkedin.metadata.service; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.linkedin.assertion.AssertionInfo; +import com.linkedin.assertion.AssertionResult; +import com.linkedin.assertion.AssertionRunEvent; +import com.linkedin.assertion.AssertionRunStatus; +import com.linkedin.assertion.AssertionSource; +import com.linkedin.assertion.AssertionSourceType; +import com.linkedin.assertion.AssertionType; +import com.linkedin.assertion.CustomAssertionInfo; +import com.linkedin.common.DataPlatformInstance; +import com.linkedin.common.EntityRelationships; +import com.linkedin.common.url.Url; +import com.linkedin.common.urn.Urn; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.AspectUtils; +import com.linkedin.metadata.graph.GraphClient; +import com.linkedin.metadata.key.AssertionKey; +import com.linkedin.metadata.query.filter.RelationshipDirection; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import io.datahubproject.metadata.context.OperationContext; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.UUID; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class AssertionService extends BaseService { + + private final GraphClient _graphClient; + + private static final String ASSERTS_RELATIONSHIP_NAME = "Asserts"; + + public AssertionService( + @Nonnull SystemEntityClient entityClient, @Nonnull GraphClient graphClient) { + super(entityClient); + _graphClient = graphClient; + } + + @Nonnull + public Urn generateAssertionUrn() { + final AssertionKey key = new AssertionKey(); + final String id = UUID.randomUUID().toString(); + key.setAssertionId(id); + return EntityKeyUtils.convertEntityKeyToUrn(key, Constants.ASSERTION_ENTITY_NAME); + } + + /** + * Retrieves the entity associated with the assertion + * + * @param opContext the operation context + * @param assertionUrn the urn of the assertion to retrieve entity for + * @return Entity urn associated with the assertion + */ + public @Nullable Urn getEntityUrnForAssertion( + @Nonnull OperationContext opContext, @Nonnull final Urn assertionUrn) { + try { + // Fetch the entity associated with the assertion from the Graph + final EntityRelationships relationships = + _graphClient.getRelatedEntities( + assertionUrn.toString(), + ImmutableList.of(ASSERTS_RELATIONSHIP_NAME), + RelationshipDirection.OUTGOING, + 0, + 1, + opContext.getActorContext().getActorUrn().toString()); + + if (relationships.hasRelationships() && !relationships.getRelationships().isEmpty()) { + return relationships.getRelationships().get(0).getEntity(); + } + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to retrieve entity for assertion with urn %s", assertionUrn), e); + } + return null; + } + + /** + * Returns an instance of {@link EntityResponse} for the specified View urn, or null if one cannot + * be found. + * + * @param assertionUrn the urn of the View + * @return an instance of {@link EntityResponse} for the View, null if it does not exist. + */ + @Nullable + public EntityResponse getAssertionEntityResponse( + @Nonnull OperationContext opContext, @Nonnull final Urn assertionUrn) { + Objects.requireNonNull(assertionUrn, "assertionUrn must not be null"); + Objects.requireNonNull(opContext, "opContext must not be null"); + try { + return this.entityClient.getV2( + opContext, + Constants.ASSERTION_ENTITY_NAME, + assertionUrn, + ImmutableSet.of( + Constants.ASSERTION_INFO_ASPECT_NAME, + Constants.ASSERTION_ACTIONS_ASPECT_NAME, + Constants.DATA_PLATFORM_INSTANCE_ASPECT_NAME, + Constants.GLOBAL_TAGS_ASPECT_NAME)); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to retrieve Assertion with urn %s", assertionUrn), e); + } + } + + public Urn upsertCustomAssertion( + @Nonnull OperationContext opContext, + @Nonnull Urn assertionUrn, + @Nonnull Urn entityUrn, + @Nonnull String description, + @Nullable String externalUrl, + @Nonnull DataPlatformInstance dataPlatformInstance, + @Nonnull CustomAssertionInfo customAssertionInfo) { + + Objects.requireNonNull(entityUrn, "entityUrn must not be null"); + Objects.requireNonNull(description, "description must not be null"); + Objects.requireNonNull(customAssertionInfo, "info must not be null"); + Objects.requireNonNull(dataPlatformInstance, "opContext must not be null"); + + AssertionInfo assertionInfo = new AssertionInfo(); + assertionInfo.setType(AssertionType.CUSTOM); + assertionInfo.setDescription(description); + if (externalUrl != null) { + assertionInfo.setExternalUrl(new Url(externalUrl)); + } + assertionInfo.setSource(new AssertionSource().setType(AssertionSourceType.EXTERNAL)); + assertionInfo.setCustomAssertion(customAssertionInfo); + + final List aspects = new ArrayList<>(); + aspects.add( + AspectUtils.buildMetadataChangeProposal( + assertionUrn, Constants.ASSERTION_INFO_ASPECT_NAME, assertionInfo)); + aspects.add( + (AspectUtils.buildMetadataChangeProposal( + assertionUrn, Constants.DATA_PLATFORM_INSTANCE_ASPECT_NAME, dataPlatformInstance))); + + try { + this.entityClient.batchIngestProposals(opContext, aspects, false); + return assertionUrn; + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to upsert Custom Assertion with urn %s", assertionUrn), e); + } + } + + public void addAssertionRunEvent( + @Nonnull OperationContext opContext, + @Nonnull Urn assertionUrn, + @Nonnull Urn asserteeUrn, + @Nonnull Long timestampMillis, + @Nonnull AssertionResult assertionResult) { + AssertionRunEvent assertionRunEvent = new AssertionRunEvent(); + assertionRunEvent.setTimestampMillis(timestampMillis); + assertionRunEvent.setRunId(timestampMillis.toString()); + assertionRunEvent.setAssertionUrn(assertionUrn); + assertionRunEvent.setAsserteeUrn(asserteeUrn); + assertionRunEvent.setStatus(AssertionRunStatus.COMPLETE); + assertionRunEvent.setResult(assertionResult); + + try { + this.entityClient.ingestProposal( + opContext, + AspectUtils.buildMetadataChangeProposal( + assertionUrn, Constants.ASSERTION_RUN_EVENT_ASPECT_NAME, assertionRunEvent), + false); + } catch (Exception e) { + throw new RuntimeException( + String.format( + "Failed to upsert Assertion Run Event for assertion with urn %s", assertionUrn), + e); + } + } +} diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/service/BaseService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/BaseService.java index 3f9022b634c67..dc533e4aa5de5 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/service/BaseService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/service/BaseService.java @@ -19,6 +19,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @@ -61,8 +62,9 @@ protected Map getTagsAspects( return finalResult; } catch (Exception e) { log.error( - "Error retrieving global tags for entities. Entities: {} aspect: {}", - entityUrns, + "Error retrieving global tags for {} entities. Sample Urns: {} aspect: {}", + entityUrns.size(), + entityUrns.stream().limit(10).collect(Collectors.toList()), Constants.GLOSSARY_TERMS_ASPECT_NAME, e); return Collections.emptyMap(); diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/service/FormService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/FormService.java index a75c399300f65..210fba82eb4e2 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/service/FormService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/service/FormService.java @@ -33,7 +33,9 @@ import com.linkedin.metadata.Constants; import com.linkedin.metadata.authorization.OwnershipUtils; import com.linkedin.metadata.entity.AspectUtils; +import com.linkedin.metadata.key.FormKey; import com.linkedin.metadata.service.util.SearchBasedFormAssignmentRunner; +import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.FormUtils; import com.linkedin.metadata.utils.SchemaFieldUtils; import com.linkedin.mxe.MetadataChangeProposal; @@ -51,6 +53,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -1050,6 +1053,28 @@ private void verifyEntityExists(@Nonnull OperationContext opContext, @Nonnull fi } } + /** Create a form given the formInfo aspect. */ + public Urn createForm( + @Nonnull OperationContext opContext, + @Nonnull final FormInfo formInfo, + @Nullable final String id) { + + FormKey formKey = new FormKey(); + String formId = id != null ? id : UUID.randomUUID().toString(); + formKey.setId(formId); + Urn formUrn = EntityKeyUtils.convertEntityKeyToUrn(formKey, FORM_ENTITY_NAME); + + try { + this.entityClient.ingestProposal( + opContext, + AspectUtils.buildMetadataChangeProposal(formUrn, FORM_INFO_ASPECT_NAME, formInfo), + false); + return formUrn; + } catch (Exception e) { + throw new RuntimeException("Failed to create form", e); + } + } + private AuditStamp createSystemAuditStamp() { return createAuditStamp(UrnUtils.getUrn(SYSTEM_ACTOR)); } diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/service/ViewService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/ViewService.java index 13bb4a5b9f73b..e01c2d224691a 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/service/ViewService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/service/ViewService.java @@ -11,12 +11,14 @@ import com.linkedin.metadata.entity.AspectUtils; import com.linkedin.metadata.key.DataHubViewKey; import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.r2.RemoteInvocationException; import com.linkedin.view.DataHubViewDefinition; import com.linkedin.view.DataHubViewInfo; import com.linkedin.view.DataHubViewType; import io.datahubproject.metadata.context.OperationContext; import java.util.Objects; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; @@ -177,6 +179,20 @@ public void deleteView(@Nonnull OperationContext opContext, @Nonnull Urn viewUrn try { this.entityClient.deleteEntity( opContext, Objects.requireNonNull(viewUrn, "viewUrn must not be null")); + + // Asynchronously delete all references to the entity (to return quickly) + CompletableFuture.runAsync( + () -> { + try { + this.entityClient.deleteEntityReferences(opContext, viewUrn); + } catch (RemoteInvocationException e) { + log.error( + String.format( + "Caught exception while attempting to clear all entity references for view with urn %s", + viewUrn), + e); + } + }); } catch (Exception e) { throw new RuntimeException(String.format("Failed to delete View with urn %s", viewUrn), e); } diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/service/util/SearchBasedFormAssignmentManager.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/util/SearchBasedFormAssignmentManager.java index 1f1ebaf520c43..8a3eb463aa15c 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/service/util/SearchBasedFormAssignmentManager.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/service/util/SearchBasedFormAssignmentManager.java @@ -18,7 +18,25 @@ public class SearchBasedFormAssignmentManager { private static final ImmutableList ENTITY_TYPES = - ImmutableList.of(Constants.DATASET_ENTITY_NAME); + ImmutableList.of( + Constants.DATASET_ENTITY_NAME, + Constants.DATA_JOB_ENTITY_NAME, + Constants.DATA_FLOW_ENTITY_NAME, + Constants.CHART_ENTITY_NAME, + Constants.DASHBOARD_ENTITY_NAME, + Constants.CORP_USER_ENTITY_NAME, + Constants.CORP_GROUP_ENTITY_NAME, + Constants.DOMAIN_ENTITY_NAME, + Constants.CONTAINER_ENTITY_NAME, + Constants.GLOSSARY_TERM_ENTITY_NAME, + Constants.GLOSSARY_NODE_ENTITY_NAME, + Constants.ML_MODEL_ENTITY_NAME, + Constants.ML_MODEL_GROUP_ENTITY_NAME, + Constants.ML_FEATURE_TABLE_ENTITY_NAME, + Constants.ML_FEATURE_ENTITY_NAME, + Constants.ML_PRIMARY_KEY_ENTITY_NAME, + Constants.DATA_PRODUCT_ENTITY_NAME, + Constants.SCHEMA_FIELD_ENTITY_NAME); public static void apply( OperationContext opContext, diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/systemmetadata/SystemMetadataService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/systemmetadata/SystemMetadataService.java index f06671ac3c314..3880ad1d8da11 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/systemmetadata/SystemMetadataService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/systemmetadata/SystemMetadataService.java @@ -40,7 +40,7 @@ List findByRegistry( List listRuns( Integer pageOffset, Integer pageSize, boolean includeSoftDeleted); - void configure(); + default void configure() {} void clear(); } diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java index accc0c8ceeb3b..6b1f484ac0a51 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java @@ -18,7 +18,7 @@ public interface TimeseriesAspectService { /** Configure the Time-Series aspect service one time at boot-up. */ - void configure(); + default void configure() {} /** * Count the number of entries using a filter diff --git a/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java b/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java index ffa0d600a2351..07f91bb52fe10 100644 --- a/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java +++ b/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java @@ -85,8 +85,7 @@ private void writeSearchCsv(WebApplicationContext ctx, PrintWriter pw) { entitySpecOpt -> { EntitySpec entitySpec = entitySpecOpt.get(); SearchRequest searchRequest = - SearchRequestHandler.getBuilder( - entitySpec, searchConfiguration, null, aspectRetriever) + SearchRequestHandler.getBuilder(entitySpec, searchConfiguration, null) .getSearchRequest( getOperationContext(ctx) .withSearchFlags( diff --git a/metadata-service/war/src/main/resources/boot/data_platforms.json b/metadata-service/war/src/main/resources/boot/data_platforms.json index 4135280cb1ac8..ab040ad2854dd 100644 --- a/metadata-service/war/src/main/resources/boot/data_platforms.json +++ b/metadata-service/war/src/main/resources/boot/data_platforms.json @@ -187,6 +187,16 @@ "logoUrl": "/assets/platforms/mysqllogo.png" } }, + { + "urn": "urn:li:dataPlatform:db2", + "aspect": { + "datasetNameDelimiter": ".", + "name": "db2", + "displayName": "DB2", + "type": "RELATIONAL_DB", + "logoUrl": "/assets/platforms/db2logo.png" + } + }, { "urn": "urn:li:dataPlatform:mariadb", "aspect": { diff --git a/metadata-service/war/src/main/resources/boot/policies.json b/metadata-service/war/src/main/resources/boot/policies.json index 454f0ba7d1163..53c773d130f32 100644 --- a/metadata-service/war/src/main/resources/boot/policies.json +++ b/metadata-service/war/src/main/resources/boot/policies.json @@ -34,7 +34,9 @@ "MANAGE_GLOBAL_OWNERSHIP_TYPES", "GET_ANALYTICS_PRIVILEGE", "CREATE_BUSINESS_ATTRIBUTE", - "MANAGE_BUSINESS_ATTRIBUTE" + "MANAGE_BUSINESS_ATTRIBUTE", + "MANAGE_STRUCTURED_PROPERTIES", + "MANAGE_DOCUMENTATION_FORMS" ], "displayName":"Root User - All Platform Privileges", "description":"Grants all platform privileges to root user.", @@ -179,7 +181,9 @@ "MANAGE_GLOBAL_OWNERSHIP_TYPES", "GET_ANALYTICS_PRIVILEGE", "CREATE_BUSINESS_ATTRIBUTE", - "MANAGE_BUSINESS_ATTRIBUTE" + "MANAGE_BUSINESS_ATTRIBUTE", + "MANAGE_STRUCTURED_PROPERTIES", + "MANAGE_DOCUMENTATION_FORMS" ], "displayName":"Admins - Platform Policy", "description":"Admins have all platform privileges.", @@ -265,7 +269,9 @@ "MANAGE_GLOBAL_ANNOUNCEMENTS", "MANAGE_GLOSSARIES", "MANAGE_TAGS", - "MANAGE_BUSINESS_ATTRIBUTE" + "MANAGE_BUSINESS_ATTRIBUTE", + "MANAGE_STRUCTURED_PROPERTIES", + "MANAGE_DOCUMENTATION_FORMS" ], "displayName":"Editors - Platform Policy", "description":"Editors can manage ingestion and view analytics.", diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java index ff740a4dfc0e0..c20b287a47141 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java @@ -147,7 +147,21 @@ public class PoliciesConfig { public static final Privilege MANAGE_CONNECTIONS_PRIVILEGE = Privilege.of( - "MANAGE_CONNECTIONS", "Manage Connections", "Manage connections to external platforms."); + "MANAGE_CONNECTIONS", + "Manage Connections", + "Manage connections to external DataHub platforms."); + + public static final Privilege MANAGE_STRUCTURED_PROPERTIES_PRIVILEGE = + Privilege.of( + "MANAGE_STRUCTURED_PROPERTIES", + "Manage Structured Properties", + "Manage structured properties in your instance."); + + public static final Privilege MANAGE_DOCUMENTATION_FORMS_PRIVILEGE = + Privilege.of( + "MANAGE_DOCUMENTATION_FORMS", + "Manage Documentation Forms", + "Manage forms assigned to assets to assist in documentation efforts."); public static final List PLATFORM_PRIVILEGES = ImmutableList.of( @@ -173,7 +187,9 @@ public class PoliciesConfig { MANAGE_GLOBAL_OWNERSHIP_TYPES, CREATE_BUSINESS_ATTRIBUTE_PRIVILEGE, MANAGE_BUSINESS_ATTRIBUTE_PRIVILEGE, - MANAGE_CONNECTIONS_PRIVILEGE); + MANAGE_CONNECTIONS_PRIVILEGE, + MANAGE_STRUCTURED_PROPERTIES_PRIVILEGE, + MANAGE_DOCUMENTATION_FORMS_PRIVILEGE); // Resource Privileges // @@ -263,7 +279,7 @@ public class PoliciesConfig { "The ability to edit any information about an entity. Super user privileges for the entity."); static final Privilege DELETE_ENTITY_PRIVILEGE = - Privilege.of("DELETE_ENTITY", "Delete", "The ability to delete the delete this entity."); + Privilege.of("DELETE_ENTITY", "Delete", "The ability to delete this entity."); static final Privilege EDIT_LINEAGE_PRIVILEGE = Privilege.of( @@ -345,6 +361,12 @@ public class PoliciesConfig { "Edit Dataset Queries", "The ability to edit the Queries for a Dataset."); + public static final Privilege EDIT_ENTITY_DATA_CONTRACT_PRIVILEGE = + Privilege.of( + "EDIT_ENTITY_DATA_CONTRACT", + "Edit Data Contract", + "The ability to edit the Data Contract for an entity."); + // Tag Privileges public static final Privilege EDIT_TAG_COLOR_PRIVILEGE = Privilege.of("EDIT_TAG_COLOR", "Edit Tag Color", "The ability to change the color of a Tag."); @@ -411,7 +433,7 @@ public class PoliciesConfig { public static final Privilege RESTORE_INDICES_PRIVILEGE = Privilege.of( "RESTORE_INDICES_PRIVILEGE", - "Restore Indicies API", + "Restore Indices API", "The ability to use the Restore Indices API."); public static final Privilege GET_TIMESERIES_INDEX_SIZES_PRIVILEGE = @@ -949,6 +971,38 @@ public class PoliciesConfig { ApiOperation.EXISTS, API_PRIVILEGE_MAP.get(ApiGroup.ENTITY).get(ApiOperation.EXISTS)) .build()) + .put( + // regular entity level permissions + MANAGE_ACCESS_TOKENS + Constants.ACCESS_TOKEN_ENTITY_NAME, + ImmutableMap.>>builder() + .put( + ApiOperation.CREATE, + Disjunctive.disjoint( + MANAGE_ACCESS_TOKENS, CREATE_ENTITY_PRIVILEGE, EDIT_ENTITY_PRIVILEGE)) + .put( + ApiOperation.READ, + Disjunctive.disjoint( + MANAGE_ACCESS_TOKENS, + VIEW_ENTITY_PAGE_PRIVILEGE, + GET_ENTITY_PRIVILEGE, + EDIT_ENTITY_PRIVILEGE, + DELETE_ENTITY_PRIVILEGE)) + .put( + ApiOperation.UPDATE, + Disjunctive.disjoint(MANAGE_ACCESS_TOKENS, EDIT_ENTITY_PRIVILEGE)) + .put( + ApiOperation.DELETE, + Disjunctive.disjoint(MANAGE_ACCESS_TOKENS, DELETE_ENTITY_PRIVILEGE)) + .put( + ApiOperation.EXISTS, + Disjunctive.disjoint( + MANAGE_ACCESS_TOKENS, + EXISTS_ENTITY_PRIVILEGE, + EDIT_ENTITY_PRIVILEGE, + DELETE_ENTITY_PRIVILEGE, + VIEW_ENTITY_PAGE_PRIVILEGE, + SEARCH_PRIVILEGE)) + .build()) .build(); /** diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/restli/NonExceptionHttpErrorResponse.java b/metadata-utils/src/main/java/com/linkedin/metadata/restli/NonExceptionHttpErrorResponse.java new file mode 100644 index 0000000000000..c99a3622d1520 --- /dev/null +++ b/metadata-utils/src/main/java/com/linkedin/metadata/restli/NonExceptionHttpErrorResponse.java @@ -0,0 +1,47 @@ +package com.linkedin.metadata.restli; + +import com.linkedin.restli.common.HttpStatus; +import com.linkedin.restli.server.RestLiServiceException; +import com.linkedin.restli.server.errors.ServiceError; + +/** + * Captures an error response (e.g. 404-not-found) that is not to be regarded as an + * exception within the server.
+ *
+ * Restli apparently requires http-error-responses to be represented by {@link + * RestLiServiceException}; thus, we need this class to specify an error response that isn't + * really an exception (in the context of the server).
+ * To highlight the unusual purpose of this exception, the name of this class is also deliberately + * unconventional (the class-name doesn't end with "Exception"). + */ +public class NonExceptionHttpErrorResponse extends RestLiServiceException { + + public NonExceptionHttpErrorResponse(HttpStatus status) { + super(status); + } + + public NonExceptionHttpErrorResponse(HttpStatus status, String message) { + super(status, message); + } + + public NonExceptionHttpErrorResponse(HttpStatus status, Throwable cause) { + super(status, cause); + } + + public NonExceptionHttpErrorResponse(HttpStatus status, String message, Throwable cause) { + super(status, message, cause); + } + + public NonExceptionHttpErrorResponse( + HttpStatus status, String message, Throwable cause, boolean writableStackTrace) { + super(status, message, cause, writableStackTrace); + } + + public NonExceptionHttpErrorResponse(ServiceError serviceError) { + super(serviceError); + } + + public NonExceptionHttpErrorResponse(ServiceError serviceError, Throwable cause) { + super(serviceError, cause); + } +} diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/restli/RestliUtil.java b/metadata-utils/src/main/java/com/linkedin/metadata/restli/RestliUtil.java index 737f79dc1c441..c9b1d5a8a82de 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/restli/RestliUtil.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/restli/RestliUtil.java @@ -79,6 +79,11 @@ public static RestLiServiceException resourceNotFoundException() { return resourceNotFoundException(null); } + @Nonnull + public static RestLiServiceException nonExceptionResourceNotFound() { + return new NonExceptionHttpErrorResponse(HttpStatus.S_404_NOT_FOUND); + } + @Nonnull public static RestLiServiceException resourceNotFoundException(@Nullable String message) { return new RestLiServiceException(HttpStatus.S_404_NOT_FOUND, message); diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java index ae061a2d0c090..7974d239a25bc 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java @@ -6,7 +6,10 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.entity.Aspect; import com.linkedin.entity.EntityResponse; +import com.linkedin.metadata.aspect.EnvelopedSystemAspect; +import com.linkedin.metadata.aspect.SystemAspect; import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.mxe.GenericAspect; import com.linkedin.mxe.GenericPayload; import java.nio.charset.StandardCharsets; @@ -87,4 +90,27 @@ public static Map> entityResponseToAspectMap( .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } + + @Nonnull + public static Map> entityResponseToSystemAspectMap( + Map inputMap, @Nonnull EntityRegistry entityRegistry) { + return inputMap.entrySet().stream() + .map( + entry -> + Map.entry( + entry.getKey(), + entry.getValue().getAspects().entrySet().stream() + .filter(aspectEntry -> aspectEntry.getValue() != null) + .map( + aspectEntry -> + Map.entry( + aspectEntry.getKey(), + EnvelopedSystemAspect.of( + entry.getKey(), + aspectEntry.getValue(), + entityRegistry.getEntitySpec( + entry.getKey().getEntityType())))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } } diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/log/LogMessageFilter.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/log/LogMessageFilter.java index 67f0ae4c77eaf..3750c0ae0bc4a 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/log/LogMessageFilter.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/log/LogMessageFilter.java @@ -1,6 +1,8 @@ package com.linkedin.metadata.utils.log; import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.IThrowableProxy; +import ch.qos.logback.classic.spi.ThrowableProxyUtil; import ch.qos.logback.core.filter.AbstractMatcherFilter; import ch.qos.logback.core.spi.FilterReply; import java.util.ArrayList; @@ -21,7 +23,21 @@ public FilterReply decide(ILoggingEvent event) { return FilterReply.NEUTRAL; } - if (this.excluded.stream().anyMatch(message -> event.getFormattedMessage().contains(message))) { + final String formattedMessage = event.getFormattedMessage(); + final IThrowableProxy throwableProxy = event.getThrowableProxy(); + + String throwableString; + if (throwableProxy != null) { + throwableString = ThrowableProxyUtil.asString(throwableProxy); + } else { + throwableString = null; + } + + if (this.excluded.stream() + .anyMatch( + message -> + formattedMessage.contains(message) + || (throwableString != null && throwableString.contains(message)))) { return FilterReply.DENY; } return FilterReply.ACCEPT; diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/metrics/ExceptionUtils.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/metrics/ExceptionUtils.java new file mode 100644 index 0000000000000..b3bad8e824218 --- /dev/null +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/metrics/ExceptionUtils.java @@ -0,0 +1,43 @@ +package com.linkedin.metadata.utils.metrics; + +import com.linkedin.metadata.aspect.plugins.validation.ValidationExceptionCollection; + +public class ExceptionUtils { + private static final String BASE_NAME = "metadata.validation.exception"; + private static final String DELIMITER = "."; + + private ExceptionUtils() {} + + public static ValidationExceptionCollection collectMetrics( + final ValidationExceptionCollection exceptions) { + exceptions + .streamAllExceptions() + .forEach( + exception -> { + String subTypeBaseName = + String.join( + DELIMITER, BASE_NAME, exception.getSubType().toString().toLowerCase()); + // subtype count + MetricUtils.counter(subTypeBaseName).inc(exceptions.size()); + // Change type count + MetricUtils.counter( + String.join( + DELIMITER, + subTypeBaseName, + exception.getChangeType().toString().toLowerCase())) + .inc(); + // Entity count + MetricUtils.counter( + String.join( + DELIMITER, + subTypeBaseName, + exception.getEntityUrn().getEntityType().toLowerCase())) + .inc(); + // Aspect count + MetricUtils.counter( + String.join(DELIMITER, subTypeBaseName, exception.getAspectName())) + .inc(); + }); + return exceptions; + } +} diff --git a/smoke-test/tests/assertions/custom_assertions_test.py b/smoke-test/tests/assertions/custom_assertions_test.py new file mode 100644 index 0000000000000..509f1cf0f04e0 --- /dev/null +++ b/smoke-test/tests/assertions/custom_assertions_test.py @@ -0,0 +1,188 @@ +import time +from typing import Any + +import pytest +from datahub.emitter.mce_builder import make_dataset_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.graph.client import ( + DatahubClientConfig, + DataHubGraph, + DataHubGraphConfig, +) +from datahub.metadata.schema_classes import StatusClass + +from tests.consistency_utils import wait_for_writes_to_sync +from tests.utils import delete_urn, get_gms_url, wait_for_healthcheck_util + +restli_default_headers = { + "X-RestLi-Protocol-Version": "2.0.0", +} + +TEST_DATASET_URN = make_dataset_urn(platform="postgres", name="foo_custom") + + +@pytest.fixture(scope="module", autouse=False) +def graph() -> DataHubGraph: + graph: DataHubGraph = DataHubGraph(config=DatahubClientConfig(server=get_gms_url())) + return graph + + +@pytest.fixture(scope="session") +def wait_for_healthchecks(): + wait_for_healthcheck_util() + mcpw = MetadataChangeProposalWrapper( + entityUrn=TEST_DATASET_URN, aspect=StatusClass(removed=False) + ) + with DataHubGraph(DataHubGraphConfig()) as graph: + graph.emit(mcpw) + yield + delete_urn(TEST_DATASET_URN) + + +def test_create_update_delete_dataset_custom_assertion( + wait_for_healthchecks: Any, graph: DataHubGraph +) -> None: + # Create custom assertion + resp = graph.upsert_custom_assertion( + urn=None, + entity_urn=TEST_DATASET_URN, + type="My custom category", + description="Description of custom assertion", + platform_name="customDQPlatform", + ) + + assert resp.get("urn") + assertion_urn = resp["urn"] + + # Update custom assertion + resp = graph.upsert_custom_assertion( + urn=assertion_urn, + entity_urn=TEST_DATASET_URN, + type="My custom category", + description="Updated Description of custom assertion", + platform_name="customDQPlatform", + external_url="http://some_url", + ) + + wait_for_writes_to_sync() + + # Report custom assertion result for success + result_reported = graph.report_assertion_result( + urn=assertion_urn, + timestamp_millis=0, + type="SUCCESS", + external_url="http://some_url/run/1", + ) + assert result_reported + + # Report custom assertion result for error + result_reported = graph.report_assertion_result( + urn=assertion_urn, + timestamp_millis=round(time.time() * 1000), + type="ERROR", + external_url="http://some_url/run/2", + error_type="SOURCE_QUERY_FAILED", + error_message="Source query failed with error Permission Denied.", + ) + assert result_reported + + # Report custom assertion result for failure + result_reported = graph.report_assertion_result( + urn=assertion_urn, + timestamp_millis=round(time.time() * 1000), + type="FAILURE", + external_url="http://some_url/run/3", + ) + assert result_reported + + wait_for_writes_to_sync() + + graphql_query_retrive_assertion = """ + query dataset($datasetUrn: String!) { + dataset(urn: $datasetUrn) { + assertions(start: 0, count: 1000) { + start + count + total + assertions { + urn + # Fetch the last run of each associated assertion. + runEvents(status: COMPLETE, limit: 3) { + total + failed + succeeded + runEvents { + timestampMillis + status + result { + type + externalUrl + nativeResults { + key + value + } + } + } + } + info { + type + description + externalUrl + lastUpdated { + time + actor + } + customAssertion { + type + entityUrn + field { + path + } + logic + } + source { + type + created { + time + actor + } + } + } + } + } + } + } + """ + + dataset_assertions = graph.execute_graphql( + query=graphql_query_retrive_assertion, + variables={"datasetUrn": TEST_DATASET_URN}, + ) + + assertions = dataset_assertions["dataset"]["assertions"]["assertions"] + assert assertions + assert assertions[0]["urn"] == assertion_urn + assert assertions[0]["info"] + assert assertions[0]["info"]["type"] == "CUSTOM" + assert assertions[0]["info"]["externalUrl"] == "http://some_url" + assert ( + assertions[0]["info"]["description"] + == "Updated Description of custom assertion" + ) + assert assertions[0]["info"]["customAssertion"] + assert assertions[0]["info"]["customAssertion"]["type"] == "My custom category" + + assert assertions[0]["runEvents"] + assert assertions[0]["runEvents"]["total"] == 3 + assert assertions[0]["runEvents"]["succeeded"] == 1 + assert assertions[0]["runEvents"]["failed"] == 1 + assert assertions[0]["runEvents"]["runEvents"][0]["result"]["externalUrl"] + + graph.delete_entity(assertion_urn, True) + + dataset_assertions = graph.execute_graphql( + query=graphql_query_retrive_assertion, + variables={"datasetUrn": TEST_DATASET_URN}, + ) + assertions = dataset_assertions["dataset"]["assertions"]["assertions"] + assert not assertions diff --git a/smoke-test/tests/cypress/cypress/e2e/auto_complete/auto_complete.js b/smoke-test/tests/cypress/cypress/e2e/auto_complete/auto_complete.js index df7d20eb6f933..4afe7b064f48e 100644 --- a/smoke-test/tests/cypress/cypress/e2e/auto_complete/auto_complete.js +++ b/smoke-test/tests/cypress/cypress/e2e/auto_complete/auto_complete.js @@ -26,12 +26,14 @@ describe("auto-complete", () => { cy.login(); cy.visit("/"); cy.get("input[data-testid=search-input]").type("SampleCypressHiveDataset"); - cy.get('[data-testid^="auto-complete-option"]').first().click(); + cy.get('[data-testid^="auto-complete-option"]') + .contains("SampleCypressHiveDataset") + .click(); + cy.get(".ant-table-row").should("be.visible"); cy.url().should( "include", "dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", ); - cy.wait(2000); }); it("should filter auto-complete results when clicking on a quick filter", () => { diff --git a/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js b/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js index 9c320eaf77ae4..ed7522eb90207 100644 --- a/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js +++ b/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js @@ -164,9 +164,19 @@ describe("Verify nested domains test functionalities", () => { clearAndDelete(); }); + it("verify Move domain root level to parent level", () => { + cy.waitTextVisible(domainName); + moveDomaintoRootLevel(); + cy.waitTextVisible("Moved Domain!"); + cy.ensureTextNotPresent("Moved Domain!"); + cy.goToDomainList(); + cy.waitTextVisible("1 sub-domain"); + }); + it("Verify Move domain parent level to root level", () => { moveDomaintoParent(); cy.waitTextVisible("Moved Domain!"); + cy.ensureTextNotPresent("Moved Domain!"); cy.goToDomainList(); cy.waitTextVisible(domainName); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/ingestion_source.js b/smoke-test/tests/cypress/cypress/e2e/mutations/ingestion_source.js index 8f50262b41d2c..470f9e2eec461 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/ingestion_source.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/ingestion_source.js @@ -13,7 +13,7 @@ describe("ingestion source creation flow", () => { cy.goToIngestionPage(); cy.clickOptionWithTestId("create-ingestion-source-button"); cy.clickOptionWithText("Snowflake"); - cy.waitTextVisible("Snowflake Recipe"); + cy.waitTextVisible("Snowflake Details"); cy.get("#account_id").type(accound_id); cy.get("#warehouse").type(warehouse_id); cy.get("#username").type(username); @@ -34,7 +34,7 @@ describe("ingestion source creation flow", () => { cy.clickOptionWithTestId("recipe-builder-next-button"); cy.waitTextVisible("Configure an Ingestion Schedule"); cy.clickOptionWithTestId("ingestion-schedule-next-button"); - cy.waitTextVisible("Give this ingestion source a name."); + cy.waitTextVisible("Give this data source a name"); cy.get('[data-testid="source-name-input"]').type(ingestion_source_name); cy.clickOptionWithTestId("ingestion-source-save-button"); cy.waitTextVisible("Successfully created ingestion source!").wait(5000); @@ -47,7 +47,7 @@ describe("ingestion source creation flow", () => { cy.get('[data-testid="ingestion-source-table-edit-button"]') .first() .click(); - cy.waitTextVisible("Edit Ingestion Source"); + cy.waitTextVisible("Edit Data Source"); cy.get("#account_id").should("have.value", accound_id); cy.get("#warehouse").should("have.value", warehouse_id); cy.get("#username").should("have.value", username); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js index d23b0ca7523b8..d01c762401c2e 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js @@ -1,40 +1,44 @@ -function readyToTypeEditor() { - return cy.get(".monaco-editor textarea:first").click().focused(); -} - -describe("run managed ingestion", () => { - it("create run managed ingestion source", () => { - const number = Math.floor(Math.random() * 100000); - const testName = `cypress test source ${number}`; - const cli_version = "0.12.0"; - cy.login(); - cy.goToIngestionPage(); - cy.clickOptionWithText("Create new source"); - cy.clickOptionWithTextToScrollintoView("Other"); - - cy.waitTextVisible("source-type"); - readyToTypeEditor().type("{ctrl}a").clear(); - readyToTypeEditor().type("source:{enter}"); - readyToTypeEditor().type(" type: demo-data"); - readyToTypeEditor().type("{enter}"); - // no space because the editor starts new line at same indentation - readyToTypeEditor().type("config: {}"); - cy.clickOptionWithText("Next"); - cy.clickOptionWithText("Next"); - - cy.enterTextInTestId("source-name-input", testName); - cy.clickOptionWithText("Advanced"); - cy.enterTextInTestId("cli-version-input", cli_version); - cy.clickOptionWithTextToScrollintoView("Save & Run"); - cy.waitTextVisible(testName); - - cy.contains(testName) - .parent() - .within(() => { - cy.contains("Succeeded", { timeout: 180000 }); - cy.clickOptionWithTestId("delete-button"); - }); - cy.clickOptionWithText("Yes"); - cy.ensureTextNotPresent(testName); - }); -}); +// TODO: Investigate why this test can never pass on CI, but passes locally after PR #21465 +// +// function readyToTypeEditor() { +// return cy.get(".monaco-editor textarea:first").click().focused(); +// } +// +// describe("run managed ingestion", () => { +// it("create run managed ingestion source", () => { +// const number = Math.floor(Math.random() * 100000); +// const testName = `cypress test source ${number}`; +// const cli_version = "0.12.0"; +// cy.login(); +// cy.goToIngestionPage(); +// cy.clickOptionWithText("Create new source"); +// cy.clickOptionWithTextToScrollintoView("Other"); +// +// cy.waitTextVisible("source-type"); +// cy.wait(10000); // waits for 5 seconds +// +// readyToTypeEditor().type("{ctrl}a").clear({ force: true }); +// readyToTypeEditor().type("source:{enter}", { force: true }); +// readyToTypeEditor().type(" type: demo-data", { force: true }); +// readyToTypeEditor().type("{enter}", { force: true }); +// // no space because the editor starts new line at same indentation +// readyToTypeEditor().type("config: {}", { force: true }); +// cy.clickOptionWithText("Next"); +// cy.clickOptionWithText("Next"); +// +// cy.enterTextInTestId("source-name-input", testName); +// cy.clickOptionWithText("Advanced"); +// cy.enterTextInTestId("cli-version-input", cli_version); +// cy.clickOptionWithTextToScrollintoView("Save & Run"); +// cy.waitTextVisible(testName); +// +// cy.contains(testName) +// .parent() +// .within(() => { +// cy.contains("Succeeded", { timeout: 180000 }); +// cy.clickOptionWithTestId("delete-button"); +// }); +// cy.clickOptionWithText("Yes"); +// cy.ensureTextNotPresent(testName); +// }); +// }); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js b/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js index 57eccc3211096..1d95c1533c93c 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js @@ -30,7 +30,7 @@ describe("managing secrets for ingestion creation", () => { cy.goToIngestionPage(); cy.get("#ingestion-create-source").click(); cy.clickOptionWithText("Snowflake"); - cy.waitTextVisible("Snowflake Recipe"); + cy.waitTextVisible("Snowflake Details"); cy.get("#account_id").type(accound_id); cy.get("#warehouse").type(warehouse_id); cy.get("#username").type(username); @@ -41,7 +41,7 @@ describe("managing secrets for ingestion creation", () => { cy.get("button").contains("Next").click(); cy.waitTextVisible("Configure an Ingestion Schedule"); cy.get("button").contains("Next").click(); - cy.waitTextVisible("Give this ingestion source a name."); + cy.waitTextVisible("Give this data source a name"); cy.get('[data-testid="source-name-input"]').type(ingestion_source_name); cy.get("button").contains("Save").click(); cy.waitTextVisible("Successfully created ingestion source!").wait(5000); @@ -69,7 +69,7 @@ describe("managing secrets for ingestion creation", () => { // Verify secret is not present during ingestion source creation for password dropdown cy.clickOptionWithText("Create new source"); cy.clickOptionWithText("Snowflake"); - cy.waitTextVisible("Snowflake Recipe"); + cy.waitTextVisible("Snowflake Details"); cy.get("#account_id").type(accound_id); cy.get("#warehouse").type(warehouse_id); cy.get("#username").type(username); @@ -90,7 +90,7 @@ describe("managing secrets for ingestion creation", () => { cy.get("button").contains("Next").click(); cy.waitTextVisible("Configure an Ingestion Schedule"); cy.get("button").contains("Next").click(); - cy.waitTextVisible("Give this ingestion source a name."); + cy.waitTextVisible("Give this data source a name"); cy.get('[data-testid="source-name-input"]').type(ingestion_source_name); cy.get("button").contains("Save").click(); cy.waitTextVisible("Successfully created ingestion source!").wait(5000); // prevent issue with missing form data diff --git a/smoke-test/tests/cypress/cypress/e2e/search/search.js b/smoke-test/tests/cypress/cypress/e2e/search/search.js index ea46b2d8d012b..cff48cbd8be71 100644 --- a/smoke-test/tests/cypress/cypress/e2e/search/search.js +++ b/smoke-test/tests/cypress/cypress/e2e/search/search.js @@ -22,9 +22,9 @@ describe("search", () => { it("can search, find a result, and visit the dataset page", () => { cy.login(); cy.visit( - "/search?filter_entity=DATASET&filter_tags=urn%3Ali%3Atag%3ACypress&page=1&query=users_created", + "/search?filter_entity=DATASET&filter_tags=urn%3Ali%3Atag%3ACypress&page=1&query=users created", ); - cy.contains("of 2 result"); + cy.contains("of 1 result"); cy.contains("Cypress"); diff --git a/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js b/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js index b6b6fea1a7d70..fb772bd7af1e7 100644 --- a/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js +++ b/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js @@ -100,10 +100,10 @@ describe("siblings", () => { cy.login(); cy.visit("/search?page=1&query=raw_orders"); - cy.contains("Showing 1 - 10 of "); + cy.contains("Showing 1 - 2 of "); - cy.get(".test-search-result").should("have.length", 5); - cy.get(".test-search-result-sibling-section").should("have.length", 5); + cy.get(".test-search-result").should("have.length", 1); + cy.get(".test-search-result-sibling-section").should("have.length", 1); cy.get(".test-search-result-sibling-section") .get(".test-mini-preview-class:contains(raw_orders)") diff --git a/smoke-test/tests/cypress/cypress/support/commands.js b/smoke-test/tests/cypress/cypress/support/commands.js index b6aeccfeb81a5..9b5065a7bdccf 100644 --- a/smoke-test/tests/cypress/cypress/support/commands.js +++ b/smoke-test/tests/cypress/cypress/support/commands.js @@ -107,7 +107,7 @@ Cypress.Commands.add("goToAccessTokenSettings", () => { Cypress.Commands.add("goToIngestionPage", () => { cy.visit("/ingestion"); - cy.waitTextVisible("Manage Ingestion"); + cy.waitTextVisible("Manage Data Sources"); }); Cypress.Commands.add("goToDataset", (urn, dataset_name) => { @@ -198,6 +198,21 @@ Cypress.Commands.add("clickOptionWithTextToScrollintoView", (text) => { cy.contains(text).scrollIntoView().click(); }); +Cypress.Commands.add("clickOptionInScrollView", (text, selector) => { + cy.get(selector).within(() => { + cy.contains(text).then((el) => { + // Scroll the element into view with options for better alignment + el[0].scrollIntoView({ block: "center", inline: "nearest" }); + + // Wrap the element for further chaining with Cypress commands + cy.wrap(el) + .should("be.visible") // Wait until the element is visible + .should("not.be.disabled") // Ensure the element is not disabled + .click({ force: true }); // Force click if necessary + }); + }); +}); + Cypress.Commands.add("deleteFromDropdown", () => { cy.openThreeDotDropdown(); cy.clickOptionWithText("Delete"); diff --git a/smoke-test/tests/dataproduct/test_dataproduct.py b/smoke-test/tests/dataproduct/test_dataproduct.py index 0d0141e9111c0..88a1c42ddca03 100644 --- a/smoke-test/tests/dataproduct/test_dataproduct.py +++ b/smoke-test/tests/dataproduct/test_dataproduct.py @@ -87,7 +87,7 @@ def create_test_data(filename: str): @pytest.fixture(scope="module", autouse=False) def ingest_cleanup_data(request): - new_file, filename = tempfile.mkstemp() + new_file, filename = tempfile.mkstemp(suffix=".json") try: create_test_data(filename) print("ingesting data products test data") diff --git a/smoke-test/tests/privileges/test_privileges.py b/smoke-test/tests/privileges/test_privileges.py index cdb5ead71aebc..c9a0b62159314 100644 --- a/smoke-test/tests/privileges/test_privileges.py +++ b/smoke-test/tests/privileges/test_privileges.py @@ -319,9 +319,8 @@ def test_privilege_to_create_and_manage_ingestion_source(): ) -@pytest.mark.skip(reason="Functionality and test needs to be validated for correctness") @pytest.mark.dependency(depends=["test_healthchecks"]) -def test_privilege_to_create_and_manage_access_tokens(): +def test_privilege_to_create_and_revoke_personal_access_tokens(): (admin_user, admin_pass) = get_admin_credentials() admin_session = login_as(admin_user, admin_pass) user_session = login_as("user", "user") @@ -345,13 +344,15 @@ def test_privilege_to_create_and_manage_access_tokens(): # Assign privileges to the new user to create and manage access tokens policy_urn = create_user_policy( - "urn:li:corpuser:user", ["MANAGE_ACCESS_TOKENS"], admin_session + "urn:li:corpuser:user", ["GENERATE_PERSONAL_ACCESS_TOKENS"], admin_session ) # Verify new user can create and manage access token(create, revoke) # Create a access token _ensure_can_create_access_token(user_session, create_access_token) + wait_for_writes_to_sync() + # List access tokens first to get token id list_access_tokens = { "query": """query listAccessTokens($input: ListAccessTokenInput!) {\n diff --git a/smoke-test/tests/structured_properties/test_structured_properties.py b/smoke-test/tests/structured_properties/test_structured_properties.py index 44beab930091d..bf1b5b1292750 100644 --- a/smoke-test/tests/structured_properties/test_structured_properties.py +++ b/smoke-test/tests/structured_properties/test_structured_properties.py @@ -176,6 +176,15 @@ def get_property_from_entity( return None +def to_es_name(property_name=None, namespace=default_namespace, qualified_name=None): + if property_name: + namespace_field = namespace.replace(".", "_") + return f"structuredProperties.{namespace_field}_{property_name}" + else: + escaped_qualified_name = qualified_name.replace(".", "_") + return f"structuredProperties.{escaped_qualified_name}" + + # @tenacity.retry( # stop=tenacity.stop_after_attempt(sleep_times), # wait=tenacity.wait_fixed(sleep_sec), @@ -406,10 +415,6 @@ def test_dataset_yaml_loader(ingest_cleanup_data, graph): def test_structured_property_search(ingest_cleanup_data, graph: DataHubGraph, caplog): - def to_es_name(property_name, namespace=default_namespace): - namespace_field = namespace.replace(".", "_") - return f"structuredProperties.{namespace_field}_{property_name}" - # Attach structured property to entity and to field field_property_name = f"deprecationDate{randint(10, 10000)}" @@ -573,28 +578,6 @@ def patch_one(prop_name, prop_value): assert actual_property_values == [property_value_other] -def test_dataset_structured_property_hard_delete(ingest_cleanup_data, graph, caplog): - property_name = f"hardDeleteTest{randint(10, 10000)}Property" - value_type = "string" - property_urn = f"urn:li:structuredProperty:{default_namespace}.{property_name}" - - create_property_definition( - property_name=property_name, graph=graph, value_type=value_type - ) - - test_property = StructuredProperties.from_datahub(graph=graph, urn=property_urn) - assert test_property is not None - - try: - graph.hard_delete_entity(urn=property_urn) - raise AssertionError("Should not be able to HARD delete structured property") - except Exception as e: - if "Hard delete of Structured Property Definitions is not supported" in str(e): - pass - else: - raise e - - def test_dataset_structured_property_soft_delete_validation( ingest_cleanup_data, graph, caplog ): @@ -685,10 +668,6 @@ def test_dataset_structured_property_soft_delete_read_mutation( def test_dataset_structured_property_soft_delete_search_filter_validation( ingest_cleanup_data, graph, caplog ): - def to_es_name(property_name, namespace=default_namespace): - namespace_field = namespace.replace(".", "_") - return f"structuredProperties.{namespace_field}_{property_name}" - # Create a test structured property dataset_property_name = f"softDeleteSearchFilter{randint(10, 10000)}" property_value = 30 @@ -744,3 +723,97 @@ def to_es_name(property_name, namespace=default_namespace): pass else: raise e + + +def test_dataset_structured_property_delete(ingest_cleanup_data, graph, caplog): + # Create property, assign value to target dataset urn + def create_property(target_dataset, prop_value): + property_name = f"hardDeleteTest{randint(10, 10000)}Property" + value_type = "string" + property_urn = f"urn:li:structuredProperty:{default_namespace}.{property_name}" + + create_property_definition( + property_name=property_name, + graph=graph, + value_type=value_type, + cardinality="SINGLE", + ) + + test_property = StructuredProperties.from_datahub(graph=graph, urn=property_urn) + assert test_property is not None + + # assign + dataset_patcher: DatasetPatchBuilder = DatasetPatchBuilder(urn=target_dataset) + dataset_patcher.set_structured_property( + StructuredPropertyUrn.make_structured_property_urn(property_urn), + prop_value, + ) + for mcp in dataset_patcher.build(): + graph.emit(mcp) + + return test_property + + # create and assign 2 structured properties with values + property1 = create_property(dataset_urns[0], "foo") + property2 = create_property(dataset_urns[0], "bar") + wait_for_writes_to_sync() + + # validate #1 & #2 properties assigned + assert get_property_from_entity( + dataset_urns[0], + property1.qualified_name, + graph=graph, + ) == ["foo"] + assert get_property_from_entity( + dataset_urns[0], + property2.qualified_name, + graph=graph, + ) == ["bar"] + + def validate_search(qualified_name, expected): + entity_urns = list( + graph.get_urns_by_filter( + extraFilters=[ + { + "field": to_es_name(qualified_name=qualified_name), + "negated": "false", + "condition": "EXISTS", + } + ] + ) + ) + assert entity_urns == expected + + # Validate search works for property #1 & #2 + validate_search(property1.qualified_name, expected=[dataset_urns[0]]) + validate_search(property2.qualified_name, expected=[dataset_urns[0]]) + + # delete the structured property #1 + graph.hard_delete_entity(urn=property1.urn) + wait_for_writes_to_sync() + + # validate property #1 deleted and property #2 remains + assert ( + get_property_from_entity( + dataset_urns[0], + property1.qualified_name, + graph=graph, + ) + is None + ) + assert get_property_from_entity( + dataset_urns[0], + property2.qualified_name, + graph=graph, + ) == ["bar"] + + # assert property 1 definition was removed + property1_definition = graph.get_aspect( + property1.urn, StructuredPropertyDefinitionClass + ) + assert property1_definition is None + + wait_for_writes_to_sync() + # Validate search works for property #1 & #2 + validate_search(property1.qualified_name, expected=[]) + validate_search(property2.qualified_name, expected=[dataset_urns[0]])