Merge branch 'master' into custom-models

.github/workflows/spark-smoke-test.yml

@@ -40,6 +40,8 @@ jobs:
           python-version: "3.7"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
+      - name: Remove images
+        run: docker image prune -a -f || true
       - name: Smoke test
         run: |
           ./gradlew :metadata-integration:java:spark-lineage:integrationTest \

build.gradle

@@ -1,7 +1,7 @@
 buildscript {
   ext.junitJupiterVersion = '5.6.1'
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
-  ext.pegasusVersion = '29.22.16'
+  ext.pegasusVersion = '29.45.0'
   ext.mavenVersion = '3.6.3'
   ext.springVersion = '5.3.29'
   ext.springBootVersion = '2.7.14'
@@ -18,6 +18,7 @@ buildscript {
   ext.hadoop3Version = '3.3.5'
   ext.kafkaVersion = '2.3.0'
   ext.hazelcastVersion = '5.3.1'
+  ext.ebeanVersion = '12.16.1'
 
   ext.docker_registry = 'linkedin'
 
@@ -29,16 +30,16 @@ buildscript {
     classpath 'io.acryl.gradle.plugin:gradle-avro-plugin:0.8.1'
     classpath 'org.springframework.boot:spring-boot-gradle-plugin:' + springBootVersion
     classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0"
-    classpath "com.palantir.gradle.gitversion:gradle-git-version:0.12.3"
-    classpath "org.gradle.playframework:gradle-playframework:0.12"
-    classpath "gradle.plugin.org.hidetake:gradle-swagger-generator-plugin:2.18.1"
+    classpath "com.palantir.gradle.gitversion:gradle-git-version:3.0.0"
+    classpath "org.gradle.playframework:gradle-playframework:0.14"
+    classpath "gradle.plugin.org.hidetake:gradle-swagger-generator-plugin:2.19.1"
   }
 }
 
 plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.0-rc2'
   id 'com.github.johnrengelman.shadow' version '6.1.0'
-  id "com.palantir.docker" version "0.35.0"
+  id 'com.palantir.docker' version '0.35.0'
   // https://blog.ltgt.net/javax-jakarta-mess-and-gradle-solution/
   // TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"
 }
@@ -86,16 +87,17 @@ project.ext.externalDependency = [
     'dgraph4j' : 'io.dgraph:dgraph4j:21.03.1',
     'dropwizardMetricsCore': 'io.dropwizard.metrics:metrics-core:4.2.3',
     'dropwizardMetricsJmx': 'io.dropwizard.metrics:metrics-jmx:4.2.3',
-    'ebean': 'io.ebean:ebean:11.33.3',
-    'ebeanAgent': 'io.ebean:ebean-agent:11.27.1',
+    'ebean': 'io.ebean:ebean:' + ebeanVersion,
+    'ebeanAgent': 'io.ebean:ebean-agent:' + ebeanVersion,
+    'ebeanDdl': 'io.ebean:ebean-ddl-generator:' + ebeanVersion,
     'elasticSearchRest': 'org.elasticsearch.client:elasticsearch-rest-high-level-client:' + elasticsearchVersion,
     'elasticSearchTransport': 'org.elasticsearch.client:transport:' + elasticsearchVersion,
     'findbugsAnnotations': 'com.google.code.findbugs:annotations:3.0.1',
     'graphqlJava': 'com.graphql-java:graphql-java:19.5',
     'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:19.1',
     'gson': 'com.google.code.gson:gson:2.8.9',
     'guice': 'com.google.inject:guice:4.2.3',
-    'guava': 'com.google.guava:guava:27.0.1-jre',
+    'guava': 'com.google.guava:guava:32.1.2-jre',
     'h2': 'com.h2database:h2:2.1.214',
     'hadoopCommon':'org.apache.hadoop:hadoop-common:2.7.2',
     'hadoopMapreduceClient':'org.apache.hadoop:hadoop-mapreduce-client-core:2.7.2',
@@ -239,7 +241,7 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
 
 subprojects {
 
-  apply plugin: 'maven'
+  apply plugin: 'maven-publish'
   apply plugin: 'com.gorylenko.gradle-git-properties'
 
   gitProperties {
@@ -253,7 +255,7 @@ subprojects {
 
   plugins.withType(JavaPlugin) {
     dependencies {
-      testCompile externalDependency.testng
+      testImplementation externalDependency.testng
       constraints {
         implementation('io.netty:netty-all:4.1.86.Final')
         implementation('org.apache.commons:commons-compress:1.21')

buildSrc/build.gradle

@@ -5,12 +5,12 @@ buildscript {
 }
 
 dependencies {
-  compile('io.acryl:json-schema-avro:0.1.5') {
+  implementation('io.acryl:json-schema-avro:0.1.5') {
     exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
     exclude group: 'com.google.guava', module: 'guava'
   }
-  compile 'com.google.guava:guava:27.0.1-jre'
-  compile 'com.fasterxml.jackson.core:jackson-databind:2.13.5'
-  compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.5'
-  compile 'commons-io:commons-io:2.11.0'
+  implementation 'com.google.guava:guava:32.1.2-jre'
+  implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.5'
+  implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.5'
+  implementation 'commons-io:commons-io:2.11.0'
 }

buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java

@@ -21,10 +21,7 @@
 import java.util.List;
 import java.util.stream.Collectors;
 import org.gradle.api.DefaultTask;
-import org.gradle.api.tasks.CacheableTask;
-import org.gradle.api.tasks.InputDirectory;
-import org.gradle.api.tasks.OutputDirectory;
-import org.gradle.api.tasks.TaskAction;
+import org.gradle.api.tasks.*;
 
 import static com.github.fge.processing.ProcessingUtil.*;
 import static org.apache.commons.io.FilenameUtils.*;
@@ -46,6 +43,7 @@ public void setInputDirectory(String inputDirectory) {
   }
 
   @InputDirectory
+  @PathSensitive(PathSensitivity.NAME_ONLY)
   public String getInputDirectory() {
    return inputDirectory;
   }

datahub-frontend/app/controllers/CentralLogoutController.java

@@ -16,7 +16,8 @@
  */
 @Slf4j
 public class CentralLogoutController extends LogoutController {
-  private static final String DEFAULT_BASE_URL_PATH = "/login";
+  private static final String AUTH_URL_CONFIG_PATH = "/login";
+  private static final String DEFAULT_BASE_URL_PATH = "/";
   private static Boolean _isOidcEnabled = false;
 
   @Inject
@@ -36,8 +37,7 @@ public CentralLogoutController(Config config) {
   public Result executeLogout(Http.Request request) {
     if (_isOidcEnabled) {
       try {
-        return Results.redirect(DEFAULT_BASE_URL_PATH)
-                .removingFromSession(request);
+        return logout(request).toCompletableFuture().get().withNewSession();
       } catch (Exception e) {
         log.error("Caught exception while attempting to perform SSO logout! It's likely that SSO integration is mis-configured.", e);
         return redirect(
@@ -47,7 +47,7 @@ public Result executeLogout(Http.Request request) {
         .withNewSession();
       }
     }
-    return Results.redirect(DEFAULT_BASE_URL_PATH)
+    return Results.redirect(AUTH_URL_CONFIG_PATH)
             .withNewSession();
   }
 }

datahub-frontend/build.gradle

@@ -1,7 +1,7 @@
 plugins {
   id "io.github.kobylynskyi.graphql.codegen" version "4.1.1"
-  id 'com.palantir.docker'
   id 'scala'
+  id 'com.palantir.docker'
 }
 
 apply from: "../gradle/versioning/versioning.gradle"
@@ -96,7 +96,7 @@ task unversionZip(type: Copy, dependsOn: [':datahub-web-react:build', dist]) {
   into "${buildDir}/docker/"
   rename "datahub-frontend-${version}.zip", "datahub-frontend.zip"
 }
-tasks.getByName("docker").dependsOn(unversionZip)
+tasks.getByPath(":datahub-frontend:docker").dependsOn(unversionZip)
 
 task cleanLocalDockerImages {
   doLast {

datahub-frontend/conf/application.conf

@@ -200,7 +200,7 @@ auth.native.enabled = ${?AUTH_NATIVE_ENABLED}
 # auth.oidc.enabled = false # (or simply omit oidc configurations)
 
 # Login session expiration time
-auth.session.ttlInHours = 720
+auth.session.ttlInHours = 24
 auth.session.ttlInHours = ${?AUTH_SESSION_TTL_HOURS}
 
 analytics.enabled = true

datahub-frontend/play.gradle

@@ -4,7 +4,7 @@ apply plugin: "org.gradle.playframework"
 project.ext.httpPort = 9001
 project.ext.playBinaryBaseName = "datahub-frontend"
 
-tasks.withType(PlayRun) {
+runPlay {
   httpPort = project.ext.httpPort
 }
 
@@ -33,8 +33,8 @@ dependencies {
     }
   }
 
-  compile project(":metadata-service:restli-client")
-  compile project(":metadata-service:auth-config")
+  implementation project(":metadata-service:restli-client")
+  implementation project(":metadata-service:auth-config")
 
   implementation externalDependency.jettyJaas
   implementation externalDependency.graphqlJava
@@ -70,15 +70,15 @@ dependencies {
   testImplementation 'no.nav.security:mock-oauth2-server:0.3.1'
   testImplementation 'org.junit-pioneer:junit-pioneer:1.9.1'
   testImplementation externalDependency.junitJupiterApi
-  testRuntime externalDependency.junitJupiterEngine
+  testRuntimeOnly externalDependency.junitJupiterEngine
 
   implementation externalDependency.slf4jApi
   compileOnly externalDependency.lombok
-  runtime externalDependency.guice
-  runtime (externalDependency.playDocs) {
+  runtimeOnly externalDependency.guice
+  runtimeOnly (externalDependency.playDocs) {
     exclude group: 'com.typesafe.akka', module: 'akka-http-core_2.12'
   }
-  runtime externalDependency.playGuice
+  runtimeOnly externalDependency.playGuice
   implementation externalDependency.log4j2Api
 
   implementation externalDependency.logbackClassic

datahub-graphql-core/build.gradle

@@ -4,25 +4,26 @@ plugins {
 apply plugin: 'java'
 
 dependencies {
-    compile project(':metadata-service:restli-client')
-    compile project(':metadata-service:auth-impl')
-    compile project(':metadata-service:auth-config')
-    compile project(':metadata-service:configuration')
-    compile project(':metadata-service:services')
-    compile project(':metadata-io')
-    compile project(':metadata-utils')
+    implementation project(':metadata-service:restli-client')
+    implementation project(':metadata-service:auth-impl')
+    implementation project(':metadata-service:auth-config')
+    implementation project(':metadata-service:configuration')
+    implementation project(':metadata-service:services')
+    implementation project(':metadata-io')
+    implementation project(':metadata-utils')
 
     implementation externalDependency.graphqlJava
     implementation externalDependency.graphqlJavaScalars
-    compile externalDependency.antlr4Runtime
-    compile externalDependency.antlr4
-    compile externalDependency.guava
+    implementation externalDependency.antlr4Runtime
+    implementation externalDependency.antlr4
+    implementation externalDependency.guava
+    implementation externalDependency.opentelemetryAnnotations
 
     implementation externalDependency.slf4jApi
     compileOnly externalDependency.lombok
     annotationProcessor externalDependency.lombok
 
-    testCompile externalDependency.mockito
+    testImplementation externalDependency.mockito
 }
 
 graphqlCodegen {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java

@@ -68,7 +68,7 @@ public Chart apply(@Nonnull final EntityResponse entityResponse) {
         result.setUrn(entityResponse.getUrn().toString());
         result.setType(EntityType.CHART);
         EnvelopedAspectMap aspectMap = entityResponse.getAspects();
-        Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
+        Long lastIngested = SystemMetadataUtils.getLastIngestedTime(aspectMap);
         result.setLastIngested(lastIngested);
 
         MappingHelper<Chart> mappingHelper = new MappingHelper<>(aspectMap, result);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/util/RunInfo.java

@@ -0,0 +1,16 @@
+package com.linkedin.datahub.graphql.types.common.mappers.util;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.Getter;
+import lombok.Setter;
+
+
+@Data
+@Setter
+@Getter
+@AllArgsConstructor
+public class RunInfo {
+  private final String id;
+  private final Long time;
+}

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/util/SystemMetadataUtils.java

@@ -3,7 +3,10 @@
 import com.linkedin.entity.EnvelopedAspectMap;
 import com.linkedin.mxe.SystemMetadata;
 
+import java.util.ArrayList;
+import java.util.List;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 import static com.linkedin.metadata.Constants.DEFAULT_RUN_ID;
 
@@ -12,19 +15,48 @@ public class SystemMetadataUtils {
   private SystemMetadataUtils() {
   }
 
-  public static Long getLastIngested(@Nonnull EnvelopedAspectMap aspectMap) {
-    Long lastIngested = null;
+  @Nullable
+  public static Long getLastIngestedTime(@Nonnull EnvelopedAspectMap aspectMap) {
+    RunInfo lastIngestionRun = getLastIngestionRun(aspectMap);
+    return lastIngestionRun != null ? lastIngestionRun.getTime() : null;
+  }
+
+  @Nullable
+  public static String getLastIngestedRunId(@Nonnull EnvelopedAspectMap aspectMap) {
+    RunInfo lastIngestionRun = getLastIngestionRun(aspectMap);
+    return lastIngestionRun != null ? lastIngestionRun.getId() : null;
+  }
+
+  /**
+   * Returns a sorted list of all of the most recent ingestion runs based on the most recent aspects present for the entity.
+   */
+  @Nonnull
+  public static List<RunInfo> getLastIngestionRuns(@Nonnull EnvelopedAspectMap aspectMap) {
+    final List<RunInfo> runs = new ArrayList<>();
     for (String aspect : aspectMap.keySet()) {
       if (aspectMap.get(aspect).hasSystemMetadata()) {
         SystemMetadata systemMetadata = aspectMap.get(aspect).getSystemMetadata();
-        if (systemMetadata.hasRunId() && !systemMetadata.getRunId().equals(DEFAULT_RUN_ID) && systemMetadata.hasLastObserved()) {
+        if (systemMetadata.hasLastRunId() && !systemMetadata.getLastRunId().equals(DEFAULT_RUN_ID) && systemMetadata.hasLastObserved()) {
           Long lastObserved = systemMetadata.getLastObserved();
-          if (lastIngested == null || lastObserved > lastIngested) {
-            lastIngested = lastObserved;
-          }
+          String runId = systemMetadata.getLastRunId();
+          RunInfo run = new RunInfo(runId, lastObserved);
+          runs.add(run);
+        } else if (systemMetadata.hasRunId() && !systemMetadata.getRunId().equals(DEFAULT_RUN_ID) && systemMetadata.hasLastObserved()) {
+          // Handle the legacy case: Check original run ids.
+          Long lastObserved = systemMetadata.getLastObserved();
+          String runId = systemMetadata.getRunId();
+          RunInfo run = new RunInfo(runId, lastObserved);
+          runs.add(run);
         }
       }
     }
-    return lastIngested;
+    runs.sort((a, b) -> Long.compare(b.getTime(), a.getTime()));
+    return runs;
+  }
+
+  @Nullable
+  private static RunInfo getLastIngestionRun(@Nonnull EnvelopedAspectMap aspectMap) {
+    List<RunInfo> runs = getLastIngestionRuns(aspectMap);
+    return !runs.isEmpty() ? runs.get(0) : null; // Just take the first, to get the most recent run.
   }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java

@@ -42,7 +42,7 @@ public static Container map(final EntityResponse entityResponse) {
     final Container result = new Container();
     final Urn entityUrn = entityResponse.getUrn();
     final EnvelopedAspectMap aspects = entityResponse.getAspects();
-    Long lastIngested = SystemMetadataUtils.getLastIngested(aspects);
+    Long lastIngested = SystemMetadataUtils.getLastIngestedTime(aspects);
     result.setLastIngested(lastIngested);
 
     result.setUrn(entityUrn.toString());

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java

@@ -67,7 +67,7 @@ public Dashboard apply(@Nonnull final EntityResponse entityResponse) {
         result.setUrn(entityResponse.getUrn().toString());
         result.setType(EntityType.DASHBOARD);
         EnvelopedAspectMap aspectMap = entityResponse.getAspects();
-        Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
+        Long lastIngested = SystemMetadataUtils.getLastIngestedTime(aspectMap);
         result.setLastIngested(lastIngested);
 
         MappingHelper<Dashboard> mappingHelper = new MappingHelper<>(aspectMap, result);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java

@@ -57,7 +57,7 @@ public DataFlow apply(@Nonnull final EntityResponse entityResponse) {
         Urn entityUrn = entityResponse.getUrn();
 
         EnvelopedAspectMap aspectMap = entityResponse.getAspects();
-        Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
+        Long lastIngested = SystemMetadataUtils.getLastIngestedTime(aspectMap);
         result.setLastIngested(lastIngested);
 
         MappingHelper<DataFlow> mappingHelper = new MappingHelper<>(aspectMap, result);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java

@@ -60,7 +60,7 @@ public DataJob apply(@Nonnull final EntityResponse entityResponse) {
         result.setType(EntityType.DATA_JOB);
 
         EnvelopedAspectMap aspectMap = entityResponse.getAspects();
-        Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
+        Long lastIngested = SystemMetadataUtils.getLastIngestedTime(aspectMap);
         result.setLastIngested(lastIngested);
 
         entityResponse.getAspects().forEach((name, aspect) -> {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatform/mappers/DataPlatformMapper.java

@@ -35,7 +35,7 @@ public DataPlatform apply(@Nonnull final EntityResponse entityResponse) {
         result.setName(dataPlatformKey.getPlatformName());
 
         EnvelopedAspectMap aspectMap = entityResponse.getAspects();
-        Long lastIngested = SystemMetadataUtils.getLastIngested(aspectMap);
+        Long lastIngested = SystemMetadataUtils.getLastIngestedTime(aspectMap);
         result.setLastIngested(lastIngested);
 
         MappingHelper<DataPlatform> mappingHelper = new MappingHelper<>(aspectMap, result);