build(misc): misc build, config, version updates

* fix spark smoke test auth
* update elasticsearch client defaults
* bump netty version
* fix check jar shell script

.github/workflows/metadata-io.yml

@@ -8,6 +8,7 @@ on:
       - "li-utils/**"
       - "metadata-models/**"
       - "metadata-io/**"
+      - ".github/workflows/metadata-io.yml"
   pull_request:
     branches:
       - "**"
@@ -16,6 +17,7 @@ on:
       - "li-utils/**"
       - "metadata-models/**"
       - "metadata-io/**"
+      - ".github/workflows/metadata-io.yml"
   release:
     types: [published]
 
@@ -52,6 +54,8 @@ jobs:
           sudo apt-get remove 'dotnet-*' azure-cli || true
           sudo rm -rf /usr/local/lib/android/ || true
           sudo docker image prune -a -f || true
+      - name: Disk Check
+        run: df -h . && docker images
       - uses: acryldata/sane-checkout-action@v3
       - name: Set up JDK 17
         uses: actions/setup-java@v4

build.gradle

@@ -391,7 +391,7 @@ subprojects {
       implementation externalDependency.annotationApi
       constraints {
         implementation("com.google.googlejavaformat:google-java-format:$googleJavaFormatVersion")
-        implementation('io.netty:netty-all:4.1.100.Final')
+        implementation('io.netty:netty-all:4.1.114.Final')
         implementation('org.apache.commons:commons-compress:1.26.0')
         implementation('org.apache.velocity:velocity-engine-core:2.3')
         implementation('org.hibernate:hibernate-validator:6.0.20.Final')

datahub-frontend/play.gradle

@@ -20,7 +20,7 @@ dependencies {
     play('com.nimbusds:nimbus-jose-jwt:8.18')
     play('com.typesafe.akka:akka-actor_2.12:2.6.20')
     play(externalDependency.jsonSmart)
-    play('io.netty:netty-all:4.1.86.Final')
+    play('io.netty:netty-all:4.1.114.Final')
     implementation(externalDependency.commonsText) {
       because("previous versions are vulnerable to CVE-2022-42889")
     }

docker/build.gradle

@@ -108,6 +108,8 @@ dockerCompose {
         environment.put "ACTIONS_EXTRA_PACKAGES", 'acryl-datahub-actions[executor] acryl-datahub-actions'
         environment.put "ACTIONS_CONFIG", 'https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml'
         environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally
+        // disabled for spark-lineage smoke-test
+        environment.put 'DATAHUB_LOCAL_COMMON_ENV', "${rootProject.project(':metadata-integration:java:spark-lineage-legacy').projectDir}/spark-smoke-test/smoke-gms.env"
 
         useComposeFiles = ['profiles/docker-compose.yml']
         projectName = 'datahub'

metadata-integration/java/datahub-client/scripts/check_jar.sh

@@ -36,10 +36,11 @@ jar -tvf $jarFile |\
       grep -v "darwin" |\
       grep -v "MetadataChangeProposal.avsc" |\
       grep -v "aix" |\
-      grep -v "com/sun/"
-      grep -v "mozilla"
-      grep -v "VersionInfo.java"
-      grep -v "mime.types"
+      grep -v "com/sun/"  |\
+      grep -v "mozilla"  |\
+      grep -v "VersionInfo.java" |\
+      grep -v "mime.types" |\
+      grep -v "com/ibm/.*"
 
 
 if [ $? -ne 0 ]; then

metadata-integration/java/spark-lineage-legacy/spark-smoke-test/smoke-gms.env

@@ -0,0 +1,2 @@
+REST_API_AUTHORIZATION_ENABLED=false
+METADATA_SERVICE_AUTH_ENABLED=false

metadata-service/configuration/src/main/resources/application.yaml

@@ -176,8 +176,8 @@ cassandra:
 elasticsearch:
   host: ${ELASTICSEARCH_HOST:localhost}
   port: ${ELASTICSEARCH_PORT:9200}
-  threadCount: ${ELASTICSEARCH_THREAD_COUNT:1}
-  connectionRequestTimeout: ${ELASTICSEARCH_CONNECTION_REQUEST_TIMEOUT:0}
+  threadCount: ${ELASTICSEARCH_THREAD_COUNT:2}
+  connectionRequestTimeout: ${ELASTICSEARCH_CONNECTION_REQUEST_TIMEOUT:5000}
   username: ${ELASTICSEARCH_USERNAME:#{null}}
   password: ${ELASTICSEARCH_PASSWORD:#{null}}
   pathPrefix: ${ELASTICSEARCH_PATH_PREFIX:#{null}}