feat(build): support jdk8 spark lineage (#9697)

.github/workflows/build-and-test.yml

@@ -83,6 +83,10 @@ jobs:
           ./gradlew :datahub-frontend:build :datahub-web-react:build --parallel
         env:
           NODE_OPTIONS: "--max-old-space-size=3072"
+      - name: Gradle compile (jdk8) for legacy Spark
+        if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
+        run: |
+          ./gradlew -PjavaClassVersionDefault=8 :metadata-integration:java:spark-lineage:compileJava
       - uses: actions/upload-artifact@v3
         if: always()
         with:

build.gradle

@@ -1,6 +1,32 @@
 buildscript {
-  ext.jdkVersion = 17
-  ext.javaClassVersion = 11
+  ext.jdkVersionDefault = 17
+  ext.javaClassVersionDefault = 11
+
+  ext.jdkVersion = { p ->
+    // If Spring 6 is present, hard dependency on jdk17
+    if (p.configurations.any { it.getDependencies().any{
+      (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6."))
+      || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test"))
+    }}) {
+      return 17
+    } else {
+      // otherwise we can use the preferred default which can be overridden with a property: -PjdkVersionDefault
+      return p.hasProperty('jdkVersionDefault') ? Integer.valueOf((String) p.getProperty('jdkVersionDefault')) : ext.jdkVersionDefault
+    }
+  }
+
+  ext.javaClassVersion = { p ->
+    // If Spring 6 is present, hard dependency on jdk17
+    if (p.configurations.any { it.getDependencies().any{
+      (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6."))
+              || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test"))
+    }}) {
+      return 17
+    } else {
+      // otherwise we can use the preferred default which can be overridden with a property: -PjavaClassVersionDefault
+      return p.hasProperty('javaClassVersionDefault') ? Integer.valueOf((String) p.getProperty('javaClassVersionDefault')) : ext.javaClassVersionDefault
+    }
+  }
 
   ext.junitJupiterVersion = '5.6.1'
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
@@ -217,6 +243,7 @@ project.ext.externalDependency = [
     'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
     'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15',
     'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46',
+    'springBootAutoconfigureJdk11': 'org.springframework.boot:spring-boot-autoconfigure:2.7.18',
     'testng': 'org.testng:testng:7.8.0',
     'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion,
     'testContainersJunit': 'org.testcontainers:junit-jupiter:' + testContainersVersion,
@@ -252,48 +279,56 @@ allprojects {
     }
   }
 
-  if (project.plugins.hasPlugin('java')
+  /**
+   * If making changes to this section also see the sections for pegasus below
+   * which use project.plugins.hasPlugin('pegasus')
+   **/
+  if (!project.plugins.hasPlugin('pegasus') && (project.plugins.hasPlugin('java')
           || project.plugins.hasPlugin('java-library')
-          || project.plugins.hasPlugin('application')
-          || project.plugins.hasPlugin('pegasus')) {
+          || project.plugins.hasPlugin('application'))) {
 
     java {
       toolchain {
-        languageVersion = JavaLanguageVersion.of(jdkVersion)
+        languageVersion = JavaLanguageVersion.of(jdkVersion(project))
       }
     }
 
     compileJava {
-      options.release = javaClassVersion
+      options.release = javaClassVersion(project)
     }
+
     tasks.withType(JavaCompile).configureEach {
       javaCompiler = javaToolchains.compilerFor {
-        languageVersion = JavaLanguageVersion.of(jdkVersion)
+        languageVersion = JavaLanguageVersion.of(jdkVersion(project))
       }
       // Puts parameter names into compiled class files, necessary for Spring 6
       options.compilerArgs.add("-parameters")
     }
 
     tasks.withType(JavaExec).configureEach {
       javaLauncher = javaToolchains.launcherFor {
-        languageVersion = JavaLanguageVersion.of(jdkVersion)
+        languageVersion = JavaLanguageVersion.of(jdkVersion(project))
       }
     }
+  }
+
+  // not duplicated, need to set this outside and inside afterEvaluate
+  afterEvaluate {
+    /**
+     * If making changes to this section also see the sections for pegasus below
+     * which use project.plugins.hasPlugin('pegasus')
+     **/
+    if (!project.plugins.hasPlugin('pegasus') && (project.plugins.hasPlugin('java')
+            || project.plugins.hasPlugin('java-library')
+            || project.plugins.hasPlugin('application'))) {
 
-    // not duplicated, need to set this outside and inside afterEvaluate
-    afterEvaluate {
       compileJava {
-        options.release = javaClassVersion
-      }
-      tasks.withType(JavaCompile).configureEach {
-        javaCompiler = javaToolchains.compilerFor {
-          languageVersion = JavaLanguageVersion.of(jdkVersion)
-        }
+        options.release = javaClassVersion(project)
       }
 
       tasks.withType(JavaExec).configureEach {
         javaLauncher = javaToolchains.launcherFor {
-          languageVersion = JavaLanguageVersion.of(jdkVersion)
+          languageVersion = JavaLanguageVersion.of(jdkVersion(project))
         }
       }
     }
@@ -368,6 +403,30 @@ subprojects {
         dataTemplateCompile externalDependency.annotationApi // support > jdk8
         restClientCompile spec.product.pegasus.restliClient
       }
+
+      java {
+        toolchain {
+          languageVersion = JavaLanguageVersion.of(jdkVersion(project))
+        }
+      }
+
+      compileJava {
+        options.release = javaClassVersion(project)
+      }
+
+      tasks.withType(JavaCompile).configureEach {
+        javaCompiler = javaToolchains.compilerFor {
+          languageVersion = JavaLanguageVersion.of(jdkVersion(project))
+        }
+        // Puts parameter names into compiled class files, necessary for Spring 6
+        options.compilerArgs.add("-parameters")
+      }
+
+      tasks.withType(JavaExec).configureEach {
+        javaLauncher = javaToolchains.launcherFor {
+          languageVersion = JavaLanguageVersion.of(jdkVersion(project))
+        }
+      }
     }
 
     afterEvaluate {
@@ -394,6 +453,16 @@ subprojects {
         dataTemplateCompile externalDependency.annotationApi // support > jdk8
         restClientCompile spec.product.pegasus.restliClient
       }
+
+      compileJava {
+        options.release = javaClassVersion(project)
+      }
+
+      tasks.withType(JavaExec).configureEach {
+        javaLauncher = javaToolchains.launcherFor {
+          languageVersion = JavaLanguageVersion.of(jdkVersion(project))
+        }
+      }
     }
   }
 }

entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java

@@ -50,22 +50,21 @@ default boolean containsDuplicateAspects() {
 
   default Map<String, Set<String>> getUrnAspectsMap() {
     return getItems().stream()
-        .map(aspect -> Map.entry(aspect.getUrn().toString(), aspect.getAspectName()))
+        .map(aspect -> Pair.of(aspect.getUrn().toString(), aspect.getAspectName()))
         .collect(
             Collectors.groupingBy(
-                Map.Entry::getKey, Collectors.mapping(Map.Entry::getValue, Collectors.toSet())));
+                Pair::getKey, Collectors.mapping(Pair::getValue, Collectors.toSet())));
   }
 
   default Map<String, Set<String>> getNewUrnAspectsMap(
       Map<String, Set<String>> existingMap, List<? extends BatchItem> items) {
     Map<String, HashSet<String>> newItemsMap =
         items.stream()
-            .map(aspect -> Map.entry(aspect.getUrn().toString(), aspect.getAspectName()))
+            .map(aspect -> Pair.of(aspect.getUrn().toString(), aspect.getAspectName()))
             .collect(
                 Collectors.groupingBy(
-                    Map.Entry::getKey,
-                    Collectors.mapping(
-                        Map.Entry::getValue, Collectors.toCollection(HashSet::new))));
+                    Pair::getKey,
+                    Collectors.mapping(Pair::getValue, Collectors.toCollection(HashSet::new))));
 
     return newItemsMap.entrySet().stream()
         .filter(

entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/GenericJsonPatch.java

@@ -4,6 +4,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.github.fge.jsonpatch.JsonPatch;
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import javax.annotation.Nonnull;
@@ -24,7 +25,7 @@ public class GenericJsonPatch {
 
   @Nonnull
   public Map<String, List<String>> getArrayPrimaryKeys() {
-    return arrayPrimaryKeys == null ? Map.of() : arrayPrimaryKeys;
+    return arrayPrimaryKeys == null ? Collections.emptyMap() : arrayPrimaryKeys;
   }
 
   @JsonIgnore

entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginFactory.java

@@ -12,6 +12,7 @@
 import io.github.classgraph.ClassInfo;
 import io.github.classgraph.MethodInfo;
 import io.github.classgraph.ScanResult;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -40,7 +41,7 @@ public static PluginFactory withCustomClasspath(
   }
 
   public static PluginFactory withConfig(@Nullable PluginConfiguration pluginConfiguration) {
-    return PluginFactory.withCustomClasspath(pluginConfiguration, List.of());
+    return PluginFactory.withCustomClasspath(pluginConfiguration, Collections.emptyList());
   }
 
   public static PluginFactory empty() {
@@ -180,7 +181,7 @@ public EntityRegistryLoadResult.PluginLoadResult getPluginLoadResult() {
   private List<AspectPayloadValidator> buildAspectPayloadValidators(
       @Nullable PluginConfiguration pluginConfiguration) {
     return pluginConfiguration == null
-        ? List.of()
+        ? Collections.emptyList()
         : applyDisable(
             build(
                 AspectPayloadValidator.class,
@@ -190,23 +191,23 @@ private List<AspectPayloadValidator> buildAspectPayloadValidators(
 
   private List<MutationHook> buildMutationHooks(@Nullable PluginConfiguration pluginConfiguration) {
     return pluginConfiguration == null
-        ? List.of()
+        ? Collections.emptyList()
         : applyDisable(
             build(MutationHook.class, pluginConfiguration.getMutationHooks(), HOOK_PACKAGES));
   }
 
   private List<MCLSideEffect> buildMCLSideEffects(
       @Nullable PluginConfiguration pluginConfiguration) {
     return pluginConfiguration == null
-        ? List.of()
+        ? Collections.emptyList()
         : applyDisable(
             build(MCLSideEffect.class, pluginConfiguration.getMclSideEffects(), HOOK_PACKAGES));
   }
 
   private List<MCPSideEffect> buildMCPSideEffects(
       @Nullable PluginConfiguration pluginConfiguration) {
     return pluginConfiguration == null
-        ? List.of()
+        ? Collections.emptyList()
         : applyDisable(
             build(MCPSideEffect.class, pluginConfiguration.getMcpSideEffects(), HOOK_PACKAGES));
   }

entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/config/PluginConfiguration.java

@@ -1,5 +1,6 @@
 package com.linkedin.metadata.aspect.plugins.config;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -11,10 +12,10 @@
 @AllArgsConstructor
 @NoArgsConstructor
 public class PluginConfiguration {
-  private List<AspectPluginConfig> aspectPayloadValidators = List.of();
-  private List<AspectPluginConfig> mutationHooks = List.of();
-  private List<AspectPluginConfig> mclSideEffects = List.of();
-  private List<AspectPluginConfig> mcpSideEffects = List.of();
+  private List<AspectPluginConfig> aspectPayloadValidators = Collections.emptyList();
+  private List<AspectPluginConfig> mutationHooks = Collections.emptyList();
+  private List<AspectPluginConfig> mclSideEffects = Collections.emptyList();
+  private List<AspectPluginConfig> mcpSideEffects = Collections.emptyList();
 
   public static PluginConfiguration EMPTY = new PluginConfiguration();
 

entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectRetriever.java

@@ -1,10 +1,12 @@
 package com.linkedin.metadata.aspect.plugins.validation;
 
+import com.google.common.collect.ImmutableSet;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.entity.Aspect;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.r2.RemoteInvocationException;
 import java.net.URISyntaxException;
+import java.util.Collections;
 import java.util.Map;
 import java.util.Set;
 import javax.annotation.Nonnull;
@@ -15,8 +17,8 @@ public interface AspectRetriever {
   @Nullable
   default Aspect getLatestAspectObject(@Nonnull final Urn urn, @Nonnull final String aspectName)
       throws RemoteInvocationException, URISyntaxException {
-    return getLatestAspectObjects(Set.of(urn), Set.of(aspectName))
-        .getOrDefault(urn, Map.of())
+    return getLatestAspectObjects(ImmutableSet.of(urn), ImmutableSet.of(aspectName))
+        .getOrDefault(urn, Collections.emptyMap())
         .get(aspectName);
   }
 

entity-registry/src/main/java/com/linkedin/metadata/models/registry/ConfigEntityRegistry.java

@@ -28,11 +28,13 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -67,7 +69,10 @@ public class ConfigEntityRegistry implements EntityRegistry {
   public ConfigEntityRegistry(Pair<Path, Path> configFileClassPathPair) throws IOException {
     this(
         DataSchemaFactory.withCustomClasspath(configFileClassPathPair.getSecond()),
-        DataSchemaFactory.getClassLoader(configFileClassPathPair.getSecond()).stream().toList(),
+        DataSchemaFactory.getClassLoader(configFileClassPathPair.getSecond())
+            .map(Stream::of)
+            .orElse(Stream.empty())
+            .collect(Collectors.toList()),
         configFileClassPathPair.getFirst());
   }
 
@@ -112,7 +117,7 @@ private static Pair<Path, Path> getFileAndClassPath(String entityRegistryRoot)
   }
 
   public ConfigEntityRegistry(InputStream configFileInputStream) {
-    this(DataSchemaFactory.getInstance(), List.of(), configFileInputStream);
+    this(DataSchemaFactory.getInstance(), Collections.emptyList(), configFileInputStream);
   }
 
   public ConfigEntityRegistry(

entity-registry/src/main/java/com/linkedin/metadata/models/registry/PatchEntityRegistry.java

@@ -32,6 +32,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -93,7 +94,10 @@ public PatchEntityRegistry(
       throws IOException, EntityRegistryException {
     this(
         DataSchemaFactory.withCustomClasspath(configFileClassPathPair.getSecond()),
-        DataSchemaFactory.getClassLoader(configFileClassPathPair.getSecond()).stream().toList(),
+        DataSchemaFactory.getClassLoader(configFileClassPathPair.getSecond())
+            .map(Stream::of)
+            .orElse(Stream.empty())
+            .collect(Collectors.toList()),
         configFileClassPathPair.getFirst(),
         registryName,
         registryVersion);

entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/EntityRegistryLoadResult.java

@@ -1,5 +1,6 @@
 package com.linkedin.metadata.models.registry.config;
 
+import java.util.Collections;
 import java.util.Set;
 import lombok.Builder;
 import lombok.Data;
@@ -23,9 +24,9 @@ public static class PluginLoadResult {
     private int mcpSideEffectCount;
     private int mclSideEffectCount;
 
-    @Builder.Default private Set<String> validatorClasses = Set.of();
-    @Builder.Default private Set<String> mutationHookClasses = Set.of();
-    @Builder.Default private Set<String> mcpSideEffectClasses = Set.of();
-    @Builder.Default private Set<String> mclSideEffectClasses = Set.of();
+    @Builder.Default private Set<String> validatorClasses = Collections.emptySet();
+    @Builder.Default private Set<String> mutationHookClasses = Collections.emptySet();
+    @Builder.Default private Set<String> mcpSideEffectClasses = Collections.emptySet();
+    @Builder.Default private Set<String> mclSideEffectClasses = Collections.emptySet();
   }
 }