Merge branch 'master' into web-react/enables-google-analytics-4

.github/actions/ci-optimization/action.yml

@@ -17,6 +17,9 @@ outputs:
   ingestion-change:
     description: "Ingestion code has changed"
     value: ${{ steps.filter.outputs.ingestion == 'true' }}
+  ingestion-base-change:
+    description: "Ingestion base image docker image has changed"
+    value: ${{ steps.filter.outputs.ingestion-base == 'true' }}
   frontend-change:
     description: "Frontend code has changed"
     value: ${{ steps.filter.outputs.frontend == 'true' }}
@@ -56,6 +59,8 @@ runs:
             - "metadata-models/**"
             - "smoke-test/**"
             - "docker/datahub-ingestion**"
+          ingestion-base:
+            - "docker/datahub-ingestion-base/**"
           docker:
             - "docker/**"
           backend:

.github/actions/docker-custom-build-and-push/action.yml

@@ -55,7 +55,7 @@ runs:
 
     # Code for testing the build when not pushing to Docker Hub.
     - name: Build and Load image for testing (if not publishing)
-      uses: docker/build-push-action@v3
+      uses: docker/build-push-action@v5
       if: ${{ inputs.publish != 'true' }}
       with:
         context: ${{ inputs.context }}
@@ -87,19 +87,19 @@ runs:
 
     # Code for building multi-platform images and pushing to Docker Hub.
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
       if: ${{ inputs.publish == 'true' }}
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
       if: ${{ inputs.publish == 'true' }}
     - name: Login to DockerHub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       if: ${{ inputs.publish == 'true' }}
       with:
         username: ${{ inputs.username }}
         password: ${{ inputs.password }}
     - name: Build and Push Multi-Platform image
-      uses: docker/build-push-action@v3
+      uses: docker/build-push-action@v5
       if: ${{ inputs.publish == 'true' }}
       with:
         context: ${{ inputs.context }}

.github/workflows/airflow-plugin.yml

@@ -51,6 +51,9 @@ jobs:
           - python-version: "3.10"
             extra_pip_requirements: 'apache-airflow==2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
+          - python-version: "3.10"
+            extra_pip_requirements: 'apache-airflow==2.9.0 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.0/constraints-3.10.txt'
+            extra_pip_extras: plugin-v2
       fail-fast: false
     steps:
       - name: Set up JDK 17

.github/workflows/publish-datahub-jars.yml

@@ -62,7 +62,7 @@ jobs:
       - name: checkout upstream repo
         run: |
           git remote add upstream https://github.com/datahub-project/datahub.git
-          git fetch upstream --tags --force
+          git fetch upstream --tags --force --filter=tree:0
       - name: publish datahub-client jar snapshot
         if: ${{ github.event_name != 'release' }}
         env:

.gitignore

@@ -126,3 +126,6 @@ metadata-service/war/bin/
 metadata-utils/bin/
 test-models/bin/
 
+datahub-executor/
+datahub-integrations-service/
+metadata-ingestion-modules/acryl-cloud

README.md

@@ -22,13 +22,13 @@ HOSTED_DOCS_ONLY-->
 </p>
 <!-- -->
 
-# DataHub: The Metadata Platform for the Modern Data Stack
+# DataHub: The Data Discovery Platform for the Modern Data Stack
 ## Built with ❤️ by <img src="https://datahubproject.io/img/acryl-logo-light-mark.png" width="25"/> [Acryl Data](https://acryldata.io) and <img src="https://datahubproject.io/img/LI-In-Bug.png" width="25"/> [LinkedIn](https://engineering.linkedin.com)
 [![Version](https://img.shields.io/github/v/release/datahub-project/datahub?include_prereleases)](https://github.com/datahub-project/datahub/releases/latest)
 [![PyPI version](https://badge.fury.io/py/acryl-datahub.svg)](https://badge.fury.io/py/acryl-datahub)
 [![build & test](https://github.com/datahub-project/datahub/workflows/build%20&%20test/badge.svg?branch=master&event=push)](https://github.com/datahub-project/datahub/actions?query=workflow%3A%22build+%26+test%22+branch%3Amaster+event%3Apush)
 [![Docker Pulls](https://img.shields.io/docker/pulls/acryldata/datahub-gms.svg)](https://hub.docker.com/r/acryldata/datahub-gms)
-[![Slack](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://slack.datahubproject.io)
+[![Slack](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://datahubproject.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme)
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md)
 [![GitHub commit activity](https://img.shields.io/github/commit-activity/m/datahub-project/datahub)](https://github.com/datahub-project/datahub/pulls?q=is%3Apr)
 [![License](https://img.shields.io/github/license/datahub-project/datahub)](https://github.com/datahub-project/datahub/blob/master/LICENSE)
@@ -40,7 +40,7 @@ HOSTED_DOCS_ONLY-->
 ---
 
 [Quickstart](https://datahubproject.io/docs/quickstart) |
-[Features](https://datahubproject.io/docs/features) |
+[Features](https://datahubproject.io/docs/) |
 [Roadmap](https://feature-requests.datahubproject.io/roadmap) |
 [Adoption](#adoption) |
 [Demo](https://demo.datahubproject.io/) |
@@ -61,7 +61,7 @@ HOSTED_DOCS_ONLY-->
 
 ## Introduction
 
-DataHub is an open-source metadata platform for the modern data stack. Read about the architectures of different metadata systems and why DataHub excels [here](https://engineering.linkedin.com/blog/2020/datahub-popular-metadata-architectures-explained). Also read our
+DataHub is an open-source data catalog for the modern data stack. Read about the architectures of different metadata systems and why DataHub excels [here](https://engineering.linkedin.com/blog/2020/datahub-popular-metadata-architectures-explained). Also read our
 [LinkedIn Engineering blog post](https://engineering.linkedin.com/blog/2019/data-hub), check out our [Strata presentation](https://speakerdeck.com/shirshanka/the-evolution-of-metadata-linkedins-journey-strata-nyc-2019) and watch our [Crunch Conference Talk](https://www.youtube.com/watch?v=OB-O0Y6OYDE). You should also visit [DataHub Architecture](docs/architecture/architecture.md) to get a better understanding of how DataHub is implemented.
 
 ## Features & Roadmap
@@ -106,7 +106,7 @@ We welcome contributions from the community. Please refer to our [Contributing G
 
 ## Community
 
-Join our [Slack workspace](https://slack.datahubproject.io) for discussions and important announcements. You can also find out more about our upcoming [town hall meetings](docs/townhalls.md) and view past recordings.
+Join our [Slack workspace](https://datahubproject.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme) for discussions and important announcements. You can also find out more about our upcoming [town hall meetings](docs/townhalls.md) and view past recordings.
 
 ## Adoption
 

build.gradle

@@ -2,12 +2,14 @@ buildscript {
   ext.jdkVersionDefault = 17
   ext.javaClassVersionDefault = 11
 
+  def springModules = ['mae-consumer', 'mce-consumer', 'pe-consumer']
+
   ext.jdkVersion = { p ->
     // If Spring 6 is present, hard dependency on jdk17
     if (p.configurations.any { it.getDependencies().any{
       (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6."))
       || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test"))
-    }}) {
+    }} || springModules.contains(p.name)) {
       return 17
     } else {
       // otherwise we can use the preferred default which can be overridden with a property: -PjdkVersionDefault
@@ -20,7 +22,7 @@ buildscript {
     if (p.configurations.any { it.getDependencies().any {
       (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6."))
               || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test"))
-    }}) {
+    }} || springModules.contains(p.name)) {
       return 17
     } else {
       // otherwise we can use the preferred default which can be overridden with a property: -PjavaClassVersionDefault
@@ -30,29 +32,29 @@ buildscript {
 
   ext.junitJupiterVersion = '5.6.1'
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
-  ext.pegasusVersion = '29.51.6'
+  ext.pegasusVersion = '29.57.0'
   ext.mavenVersion = '3.6.3'
-  ext.springVersion = '6.1.4'
-  ext.springBootVersion = '3.2.3'
-  ext.springKafkaVersion = '3.1.2'
+  ext.springVersion = '6.1.5'
+  ext.springBootVersion = '3.2.6'
+  ext.springKafkaVersion = '3.1.6'
   ext.openTelemetryVersion = '1.18.0'
   ext.neo4jVersion = '5.14.0'
   ext.neo4jTestVersion = '5.14.0'
   ext.neo4jApocVersion = '5.14.0'
   ext.testContainersVersion = '1.17.4'
   ext.elasticsearchVersion = '2.11.1' // ES 7.10, Opensearch 1.x, 2.x
   ext.jacksonVersion = '2.15.3'
-  ext.jettyVersion = '11.0.19'
-  ext.playVersion = '2.8.21'
-  ext.log4jVersion = '2.19.0'
+  ext.jettyVersion = '11.0.21'
+  ext.playVersion = '2.8.22'
+  ext.log4jVersion = '2.23.1'
   ext.slf4jVersion = '1.7.36'
   ext.logbackClassic = '1.4.14'
   ext.hadoop3Version = '3.3.5'
   ext.kafkaVersion = '5.5.15'
   ext.hazelcastVersion = '5.3.6'
   ext.ebeanVersion = '12.16.1'
   ext.googleJavaFormatVersion = '1.18.1'
-  ext.openLineageVersion = '1.5.0'
+  ext.openLineageVersion = '1.16.0'
   ext.logbackClassicJava8 = '1.2.12'
 
   ext.docker_registry = 'acryldata'
@@ -158,17 +160,20 @@ project.ext.externalDependency = [
     'javatuples': 'org.javatuples:javatuples:1.2',
     'javaxInject' : 'javax.inject:javax.inject:1',
     'javaxValidation' : 'javax.validation:validation-api:2.0.1.Final',
+    'jakartaValidation': 'jakarta.validation:jakarta.validation-api:3.1.0-M2',
     'jerseyCore': 'org.glassfish.jersey.core:jersey-client:2.41',
     'jerseyGuava': 'org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1',
     'jettyJaas': "org.eclipse.jetty:jetty-jaas:$jettyVersion",
     'jettyClient': "org.eclipse.jetty:jetty-client:$jettyVersion",
     'jettison': 'org.codehaus.jettison:jettison:1.5.4',
     'jgrapht': 'org.jgrapht:jgrapht-core:1.5.1',
     'jna': 'net.java.dev.jna:jna:5.12.1',
-    'jsonPatch': 'com.github.java-json-tools:json-patch:1.13',
+    'jsonPatch': 'jakarta.json:jakarta.json-api:2.1.3',
+    'jsonPathImpl': 'org.eclipse.parsson:parsson:1.1.6',
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'jsonSmart': 'net.minidev:json-smart:2.4.9',
     'json': 'org.json:json:20231013',
+    'jsonSchemaValidator': 'com.github.java-json-tools:json-schema-validator:2.2.14',
     'junit': 'junit:junit:4.13.2',
     'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion",
     'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion",
@@ -245,8 +250,10 @@ project.ext.externalDependency = [
     'springBootStarterValidation': "org.springframework.boot:spring-boot-starter-validation:$springBootVersion",
     'springKafka': "org.springframework.kafka:spring-kafka:$springKafkaVersion",
     'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
+    'springRetry': "org.springframework.retry:spring-retry:2.0.6",
     'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15',
     'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46',
+    'swaggerCore': 'io.swagger.core.v3:swagger-core:2.2.7',
     'springBootAutoconfigureJdk11': 'org.springframework.boot:spring-boot-autoconfigure:2.7.18',
     'testng': 'org.testng:testng:7.8.0',
     'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion,
@@ -265,7 +272,8 @@ project.ext.externalDependency = [
     'jline':'jline:jline:1.4.1',
     'jetbrains':' org.jetbrains.kotlin:kotlin-stdlib:1.6.0',
     'annotationApi': 'javax.annotation:javax.annotation-api:1.3.2',
-    'classGraph': 'io.github.classgraph:classgraph:4.8.165',
+    'jakartaAnnotationApi': 'jakarta.annotation:jakarta.annotation-api:3.0.0',
+    'classGraph': 'io.github.classgraph:classgraph:4.8.172',
 ]
 
 allprojects {

buildSrc/build.gradle

@@ -14,15 +14,16 @@ dependencies {
    * Other companies are also separately maintaining forks (like: https://github.com/java-json-tools/json-schema-avro).
    * We have built several customizations on top of it for various bug fixes, especially around union scheams
    */
-  implementation('io.acryl:json-schema-avro:0.2.2') {
+  implementation('io.acryl:json-schema-avro:0.2.3') {
     exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
     exclude group: 'com.google.guava', module: 'guava'
   }
+  implementation 'com.github.java-json-tools:json-schema-validator:2.2.14'
   implementation 'com.google.guava:guava:32.1.2-jre'
   implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.5'
   implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.5'
   implementation 'commons-io:commons-io:2.11.0'
 
   compileOnly 'org.projectlombok:lombok:1.18.30'
   annotationProcessor 'org.projectlombok:lombok:1.18.30'
-}
+}

buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java

@@ -60,6 +60,7 @@ public class OpenApiEntities {
                 .add("dataProductProperties")
                 .add("institutionalMemory")
                 .add("forms").add("formInfo").add("dynamicFormAssignment")
+                .add("businessAttributeInfo")
                 .build();
 
     private final static ImmutableSet<String> ENTITY_EXCLUSIONS = ImmutableSet.<String>builder()

datahub-frontend/app/auth/AuthModule.java

@@ -15,6 +15,7 @@
 import com.google.inject.name.Named;
 import com.linkedin.entity.client.SystemEntityClient;
 import com.linkedin.entity.client.SystemRestliEntityClient;
+import com.linkedin.metadata.models.registry.EmptyEntityRegistry;
 import com.linkedin.metadata.restli.DefaultRestliClientFactory;
 import com.linkedin.parseq.retry.backoff.ExponentialBackoff;
 import com.linkedin.util.Configuration;
@@ -61,6 +62,8 @@ public class AuthModule extends AbstractModule {
   private static final String PAC4J_SESSIONSTORE_PROVIDER_CONF = "pac4j.sessionStore.provider";
   private static final String ENTITY_CLIENT_RETRY_INTERVAL = "entityClient.retryInterval";
   private static final String ENTITY_CLIENT_NUM_RETRIES = "entityClient.numRetries";
+  private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE = "entityClient.restli.get.batchSize";
+  private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY = "entityClient.restli.get.batchConcurrency";
   private static final String GET_SSO_SETTINGS_ENDPOINT = "auth/getSsoSettings";
 
   private final com.typesafe.config.Config _configs;
@@ -112,7 +115,7 @@ protected void configure() {
           .toConstructor(
               SsoCallbackController.class.getConstructor(
                   SsoManager.class,
-                  Authentication.class,
+                  OperationContext.class,
                   SystemEntityClient.class,
                   AuthServiceClient.class,
                   org.pac4j.core.config.Config.class,
@@ -164,8 +167,9 @@ protected Authentication provideSystemAuthentication() {
   @Provides
   @Singleton
   @Named("systemOperationContext")
-  protected OperationContext provideOperationContext(final Authentication systemAuthentication,
-                                                     final ConfigurationProvider configurationProvider) {
+  protected OperationContext provideOperationContext(
+          final Authentication systemAuthentication,
+          final ConfigurationProvider configurationProvider) {
     ActorContext systemActorContext =
             ActorContext.builder()
                     .systemAuth(true)
@@ -180,7 +184,7 @@ protected OperationContext provideOperationContext(final Authentication systemAu
             .operationContextConfig(systemConfig)
             .systemActorContext(systemActorContext)
             .searchContext(SearchContext.EMPTY)
-            .entityRegistryContext(EntityRegistryContext.EMPTY)
+            .entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY))
             // Authorizer.EMPTY doesn't actually apply to system auth
             .authorizerContext(AuthorizerContext.builder().authorizer(Authorizer.EMPTY).build())
             .build(systemAuthentication);
@@ -199,12 +203,14 @@ protected ConfigurationProvider provideConfigurationProvider() {
   protected SystemEntityClient provideEntityClient(
       @Named("systemOperationContext") final OperationContext systemOperationContext,
       final ConfigurationProvider configurationProvider) {
+
     return new SystemRestliEntityClient(
-            systemOperationContext,
         buildRestliClient(),
         new ExponentialBackoff(_configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)),
         _configs.getInt(ENTITY_CLIENT_NUM_RETRIES),
-        configurationProvider.getCache().getClient().getEntityClient());
+        configurationProvider.getCache().getClient().getEntityClient(),
+        Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE)),
+        Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY)));
   }
 
   @Provides