Merge pull request #115 from neojunjie/sync_v0.10.3

Sync v0.10.3

.github/actions/docker-custom-build-and-push/action.yml

@@ -23,6 +23,9 @@ inputs:
     # e.g. linkedin/datahub-gms
     description: "List of Docker images to use as base name for tags"
     required: true
+  build-args:
+    description: "List of build-time variables. Same as docker/build-push-action"
+    required: false
   tags:
     # e.g. latest,head,sha12345
     description: "List of tags to use for the Docker image"
@@ -49,14 +52,15 @@ runs:
 
     # Code for testing the build when not pushing to Docker Hub.
     - name: Build and Load image for testing (if not publishing)
-      uses: docker/build-push-action@v2
+      uses: docker/build-push-action@v3
       if: ${{ inputs.publish != 'true' }}
       with:
         context: ${{ inputs.context }}
         file: ${{ inputs.file }}
         # TODO this only does single-platform builds in testing?
         # leaving it for now since it matches the previous behavior
         platforms: linux/amd64
+        build-args: ${{ inputs.build-args }}
         tags: ${{ steps.docker_meta.outputs.tags }}
         load: true
         push: false
@@ -68,24 +72,25 @@ runs:
 
     # Code for building multi-platform images and pushing to Docker Hub.
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
+      uses: docker/setup-qemu-action@v2
       if: ${{ inputs.publish == 'true' }}
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v2
       if: ${{ inputs.publish == 'true' }}
     - name: Login to DockerHub
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       if: ${{ inputs.publish == 'true' }}
       with:
         username: ${{ inputs.username }}
         password: ${{ inputs.password }}
     - name: Build and Push Multi-Platform image
-      uses: docker/build-push-action@v2
+      uses: docker/build-push-action@v3
       if: ${{ inputs.publish == 'true' }}
       with:
         context: ${{ inputs.context }}
         file: ${{ inputs.file }}
         platforms: ${{ inputs.platforms }}
+        build-args: ${{ inputs.build-args }}
         tags: ${{ steps.docker_meta.outputs.tags }}
         push: true
 

.github/actions/ensure-codegen-updated/action.yml

@@ -10,7 +10,6 @@ runs:
           # See https://unix.stackexchange.com/a/155077/378179.
           echo 'There are uncommitted changes:'
           echo $output
-          exit 1
         else
           echo 'All good!'
         fi

.github/pr-labeler-config.yml

@@ -1,14 +1,22 @@
 ingestion:
-- 'metadata-ingestion/**/*'
+  - "metadata-ingestion/**/*"
+  - "metadata-ingestion-modules/**/*"
+  - "metadata-integration/**/*"
 
 devops:
-- 'docker/**/*'
-- '.github/**/*'
+  - "docker/**/*"
+  - ".github/**/*"
+  - "perf-test/**/*"
+  - "metadata-service/**/*"
 
 product:
-- 'datahub-web-react/**/*'
-- 'datahub-frontend/**/*'
-- 'datahub-graphql-core/**/*'
+  - "datahub-web-react/**/*"
+  - "datahub-frontend/**/*"
+  - "datahub-graphql-core/**/*"
+  - "metadata-io/**/*"
 
 docs:
-- 'docs/**/*'
+  - "docs/**/*"
+
+smoke_test:
+  - "smoke-test/**/*"

.github/pull_request_template.md

@@ -1,8 +1,8 @@
 
-
 ## Checklist
+
 - [ ] The PR conforms to DataHub's [Contributing Guideline](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md) (particularly [Commit Message Format](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md#commit-message-format))
 - [ ] Links to related issues (if applicable)
 - [ ] Tests for the changes have been added/updated (if applicable)
 - [ ] Docs related to the changes have been added/updated (if applicable). If a new feature has been added a Usage Guide has been added for the same.
-- [ ] For any breaking change/potential downtime/deprecation/big changes an entry has been made in [Updating DataHub](https://github.com/datahub-project/datahub/blob/master/docs/how/updating-datahub.md)
+- [ ] For any breaking change/potential downtime/deprecation/big changes an entry has been made in [Updating DataHub](https://github.com/datahub-project/datahub/blob/master/docs/how/updating-datahub.md)

.github/scripts/check_event_type.py

@@ -0,0 +1,23 @@
+import sys
+
+java_events = set()
+with open("./metadata-io/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java") as java_file:
+    for line in java_file:
+        if '''Event"''' not in line:
+            continue
+        line = line.replace("\n", "").split('"')[1]
+        java_events.add(line)
+
+ts_events = set()
+with open("././datahub-web-react/src/app/analytics/event.ts") as ts_file:
+    for line in ts_file:
+        if '''Event,''' not in line:
+            continue
+        line = line.replace(",\n", "").replace(" ", "")
+        ts_events.add(line)
+
+ts_events_not_in_java = ts_events.difference(java_events)
+if len(ts_events_not_in_java) > 0:
+    print(f"Missing {ts_events_not_in_java} from DataHubUsageEventType.java. Please add")
+    sys.exit(1)
+print("Passed")

.github/scripts/check_policies.py

@@ -0,0 +1,63 @@
+import json
+import pprint
+
+with open(
+    "./metadata-service/war/src/main/resources/boot/policies.json"
+) as policies_file:
+    all_policies = json.loads(policies_file.read())
+
+metadata_policies = []
+platform_policies = []
+other_policies = []
+without_info = []
+
+metadata_privileges = set()
+platform_privileges = set()
+for policy in all_policies:
+    urn = policy["urn"]
+    if urn == "urn:li:dataHubPolicy:0":
+        root_user_platform_policy_privileges = policy["info"]["privileges"]
+    elif urn == "urn:li:dataHubPolicy:editor-platform-policy":
+        editor_platform_policy_privileges = policy["info"]["privileges"]
+    elif urn == "urn:li:dataHubPolicy:7":
+        all_user_platform_policy_privilges = policy["info"]["privileges"]
+    try:
+        doc_type = policy["info"]["type"]
+        privileges = policy["info"]["privileges"]
+        if doc_type == "METADATA":
+            metadata_policies.append(policy)
+            metadata_privileges.update(privileges)
+        elif doc_type == "PLATFORM":
+            platform_policies.append(policy)
+            platform_privileges.update(privileges)
+        else:
+            other_policies.append(policy)
+    except:
+        without_info.append(policy)
+        pprint.pprint(policy)
+
+print(
+    f"""
+    Number of policies is {len(all_policies)}
+    Number of metadata_policies is {len(metadata_policies)}
+    Number of platform_policies is {len(platform_policies)}
+    Number of other is {len(other_policies)}
+    Number without info is {len(without_info)}
+
+    Number metadata privileges are {len(metadata_privileges)}
+    Number platform privileges are {len(platform_privileges)}
+"""
+)
+
+diff_policies = set(platform_privileges).difference(
+    set(root_user_platform_policy_privileges)
+)
+assert len(diff_policies) == 0, f"Missing privileges for root user are {diff_policies}"
+
+diff_policies = set(editor_platform_policy_privileges).difference(
+    set(all_user_platform_policy_privilges)
+)
+assert "MANAGE_POLICIES" not in all_user_platform_policy_privilges
+assert (
+    len(diff_policies) == 0
+), f"Missing privileges for all user policies are {diff_policies}"

.github/scripts/docker_helpers.sh

@@ -0,0 +1,24 @@
+echo "GITHUB_REF: $GITHUB_REF"
+echo "GITHUB_SHA: $GITHUB_SHA"
+
+export MAIN_BRANCH="master"
+export MAIN_BRANCH_TAG="head"
+
+function get_short_sha {
+    echo $(git rev-parse --short "$GITHUB_SHA")
+}
+
+export SHORT_SHA=$(get_short_sha)
+echo "SHORT_SHA: $SHORT_SHA"
+
+function get_tag {
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+}
+
+function get_python_docker_release_v {
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},0.0.0+docker.${SHORT_SHA},g" -e 's,refs/tags/v\(.*\),\1+docker,g' -e 's,refs/pull/\([0-9]*\).*,0.0.0+docker.pr\1,g')
+}
+
+function get_unique_tag {
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+}

.github/workflows/build-and-test.yml

@@ -13,29 +13,51 @@ on:
       - "docs/**"
       - "**.md"
   release:
-    types: [published, edited]
+    types: [published]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
   build:
+    strategy:
+      fail-fast: false
+      matrix:
+        command:
+          [
+            "./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test -x :metadata-ingestion-modules:airflow-plugin:build -x :datahub-frontend:build -x :datahub-web-react:build --parallel",
+            "./gradlew :datahub-frontend:build :datahub-web-react:build --parallel",
+            "./gradlew :metadata-ingestion-modules:airflow-plugin:build --parallel"
+          ]
+        timezone:
+          [
+            "UTC",
+            "America/New_York",
+          ]
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
-        uses: actions/setup-java@v1
+      - uses: szenius/[email protected]
         with:
-          java-version: 1.8
-      - uses: actions/setup-python@v2
+          timezoneLinux: ${{ matrix.timezone }}
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: 11
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.7"
       - name: Gradle build (and test)
         run: |
-          ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test
-      - uses: actions/upload-artifact@v2
+          ${{ matrix.command }}
+        env:
+          NODE_OPTIONS: "--max-old-space-size=3072"
+      - uses: actions/upload-artifact@v3
         if: always()
         with:
           name: Test Results (build)
@@ -56,18 +78,22 @@ jobs:
   quickstart-compose-validation:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.7"
+      - name: Download YQ
+        uses: chrisdickinson/[email protected]
+        with:
+          yq-version: v4.28.2
       - name: Quickstart Compose Validation
         run: ./docker/quickstart/generate_and_compare.sh
 
   event-file:
     runs-on: ubuntu-latest
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: Event File
           path: ${{ github.event_path }}

.github/workflows/check-datahub-jars.yml

@@ -16,36 +16,38 @@ on:
       - "docs/**"
       - "**.md"
   release:
-    types: [published, edited]
+    types: [published]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
-
   check_jars:
+    strategy:
+      max-parallel: 1
+      fail-fast: false
+      matrix:
+        command:
+          [
+            "datahub-client",
+            "datahub-protobuf",
+            "spark-lineage"
+          ]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Set up JDK 1.8
-        uses: actions/setup-java@v1
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
         with:
-          java-version: 1.8
-      - uses: actions/setup-python@v2
+          distribution: "zulu"
+          java-version: 11
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.7"
-      - name: check datahub-client jar
-        run: |
-          ./gradlew :metadata-integration:java:datahub-client:build
-          ./gradlew :metadata-integration:java:datahub-client:javadoc
-      - name: check spark-lineage jar
-        run: |
-          ./gradlew :metadata-integration:java:spark-lineage:build
-          ./gradlew :metadata-integration:java:spark-lineage:javadoc
-      - name: check datahub-protobuf jar
+      - name: check ${{ matrix.command }} jar
         run: |
-          ./gradlew :metadata-integration:java:datahub-protobuf:build
-          ./gradlew :metadata-integration:java:datahub-protobuf:javadoc
+          ./gradlew :metadata-integration:java:${{ matrix.command }}:build --info
+          ./gradlew :metadata-integration:java:${{ matrix.command }}:javadoc