fix(build): set @cliMajorVersion@ correctly (#9228)

datahub-project · Nov 11, 2023 · 0e3efab · 0e3efab
1 parent bb73002
commit 0e3efab
Show file tree

Hide file tree

Showing 12 changed files with 39 additions and 73 deletions.
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -27,7 +27,7 @@ jobs:
         command: [
             # metadata-ingestion and airflow-plugin each have dedicated build jobs
             "except_metadata_ingestion",
-            "frontend"
+            "frontend",
           ]
         timezone: ["UTC", "America/New_York"]
     runs-on: ubuntu-latest
@@ -36,9 +36,7 @@ jobs:
       - uses: szenius/[email protected]
         with:
           timezoneLinux: ${{ matrix.timezone }}
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+      - uses: hsheth2/sane-checkout-action@v1
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:

diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
@@ -27,9 +27,7 @@ jobs:
         command: ["datahub-client", "datahub-protobuf", "spark-lineage"]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+      - uses: hsheth2/sane-checkout-action@v1
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:

diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
@@ -31,9 +31,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/.github/workflows/docker-ingestion-smoke.yml b/.github/workflows/docker-ingestion-smoke.yml
@@ -50,9 +50,7 @@ jobs:
     if: ${{ needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:

diff --git a/.github/workflows/docker-postgres-setup.yml b/.github/workflows/docker-postgres-setup.yml
@@ -46,9 +46,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:

diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -74,9 +74,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :metadata-service:war:build -x test --parallel
@@ -132,9 +130,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :metadata-jobs:mae-consumer-job:build -x test --parallel
@@ -190,9 +186,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :metadata-jobs:mce-consumer-job:build -x test --parallel
@@ -248,9 +242,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :datahub-upgrade:build -x test --parallel
@@ -306,9 +298,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :datahub-frontend:dist -x test -x yarnTest -x yarnLint --parallel
@@ -366,9 +356,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
@@ -388,9 +376,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
@@ -410,9 +396,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
@@ -434,9 +418,7 @@ jobs:
     needs: setup
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -468,9 +450,7 @@ jobs:
     needs: [setup, datahub_ingestion_base_build]
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -510,9 +490,7 @@ jobs:
     needs: [setup, datahub_ingestion_base_build]
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -554,9 +532,7 @@ jobs:
     needs: [setup, datahub_ingestion_base_slim_build]
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -637,9 +613,7 @@ jobs:
     needs: [setup, datahub_ingestion_base_full_build]
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+        uses: hsheth2/sane-checkout-action@v1
       - uses: dorny/paths-filter@v2
         id: filter
         with:

diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml
@@ -48,9 +48,7 @@ jobs:
     needs: ["check-secret", "setup"]
     if: ${{ needs.check-secret.outputs.publish-enabled == 'true' }}
     steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
+      - uses: hsheth2/sane-checkout-action@v1
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:

diff --git a/.github/workflows/spark-smoke-test.yml b/.github/workflows/spark-smoke-test.yml
@@ -29,10 +29,7 @@ jobs:
   spark-smoke-test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 800
-          fetch-tags: true
+      - uses: hsheth2/sane-checkout-action@v1
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:

diff --git a/metadata-service/configuration/build.gradle b/metadata-service/configuration/build.gradle
@@ -1,6 +1,7 @@
 plugins {
     id 'java'
 }
+apply from: "../../gradle/versioning/versioning.gradle"
 
 dependencies {
     implementation externalDependency.jacksonDataBind
@@ -12,4 +13,8 @@ dependencies {
     compileOnly externalDependency.lombok
 
     annotationProcessor externalDependency.lombok
-}
+}
+
+processResources.configure {
+  finalizedBy printVersionDetails // always print version details
+}
diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml
@@ -44,6 +44,7 @@ authorization:
   restApiAuthorization: ${REST_API_AUTHORIZATION_ENABLED:false}
 
 ingestion:
+  # The value of cliMajorVersion is substituted in by the processResources Gradle task.
   enabled: ${UI_INGESTION_ENABLED:true}
   defaultCliVersion: "${UI_INGESTION_DEFAULT_CLI_VERSION:@cliMajorVersion@}"
   maxSerializedStringLength: "${INGESTION_MAX_SERIALIZED_STRING_LENGTH:16000000}" # Indicates the maximum allowed JSON String length Jackson will handle, impacts the maximum size of ingested aspects
@@ -74,7 +75,7 @@ datahub:
   plugin:
     pluginSecurityMode: ${PLUGIN_SECURITY_MODE:RESTRICTED} # Possible value RESTRICTED or LENIENT, default to RESTRICTED
     entityRegistry:
-        path: ${ENTITY_REGISTRY_PLUGIN_PATH:/etc/datahub/plugins/models}
+      path: ${ENTITY_REGISTRY_PLUGIN_PATH:/etc/datahub/plugins/models}
     retention:
       path: ${RETENTION_PLUGIN_PATH:/etc/datahub/plugins/retention}
     auth:
@@ -280,22 +281,20 @@ updateIndices:
 ingestionScheduler:
   enabled: ${ENABLE_INGESTION_SCHEDULER_HOOK:true} # enable to execute ingestion scheduling
 
-
 bootstrap:
   upgradeDefaultBrowsePaths:
     enabled: ${UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED:false} # enable to run the upgrade to migrate legacy default browse paths to new ones
   backfillBrowsePathsV2:
     enabled: ${BACKFILL_BROWSE_PATHS_V2:false} # Enables running the backfill of browsePathsV2 upgrade step. There are concerns about the load of this step so hiding it behind a flag. Deprecating in favor of running through SystemUpdate
   reprocessDefaultBrowsePathsV2:
-    enabled:  ${REPROCESS_DEFAULT_BROWSE_PATHS_V2:false} # reprocess V2 browse paths which were set to the default: {"path":[{"id":"Default"}]}
+    enabled: ${REPROCESS_DEFAULT_BROWSE_PATHS_V2:false} # reprocess V2 browse paths which were set to the default: {"path":[{"id":"Default"}]}
   policies:
     file: ${BOOTSTRAP_POLICIES_FILE:classpath:boot/policies.json}
     # eg for local file
     # file: "file:///datahub/datahub-gms/resources/custom-policies.json"
   servlets:
     waitTimeout: ${BOOTSTRAP_SERVLETS_WAITTIMEOUT:60} # Total waiting time in seconds for servlets to initialize
 
-
 systemUpdate:
   initialBackOffMs: ${BOOTSTRAP_SYSTEM_UPDATE_INITIAL_BACK_OFF_MILLIS:5000}
   maxBackOffs: ${BOOTSTRAP_SYSTEM_UPDATE_MAX_BACK_OFFS:50}
@@ -371,4 +370,4 @@ cache:
           corpUserCredentials: 20
           corpUserSettings: 20
 
-springdoc.api-docs.groups.enabled: true
+springdoc.api-docs.groups.enabled: true
diff --git a/metadata-service/factories/build.gradle b/metadata-service/factories/build.gradle
@@ -1,5 +1,4 @@
 apply plugin: 'java-library'
-apply from: "../../gradle/versioning/versioning.gradle"
 
 dependencies {
   api project(':metadata-io')
@@ -65,7 +64,3 @@ configurations.all{
   exclude group: "commons-io", module:"commons-io"
   exclude group: "jline", module:"jline"
 }
-
-processResources.configure {
-  finalizedBy printVersionDetails // always print version details
-}
diff --git a/smoke-test/tests/read_only/test_services_up.py b/smoke-test/tests/read_only/test_services_up.py
@@ -23,3 +23,11 @@ def test_gms_config_accessible():
         assert gms_config["versions"]["linkedin/datahub"]["version"] == DATAHUB_VERSION
     else:
         print("[WARN] TEST_DATAHUB_VERSION is not set")
+
+    # Make sure that the default CLI version gets generated properly.
+    # While we don't want to hardcode the actual value, we can make
+    # sure it mostly looks like a version string.
+    default_cli_version: str = gms_config["managedIngestion"]["defaultCliVersion"]
+    print(f"Default CLI version: {default_cli_version}")
+    assert not default_cli_version.startswith("@")
+    assert "." in default_cli_version