diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 060d345a6b7d9..6b7f2b5035c25 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -83,6 +83,10 @@ jobs: ./gradlew :datahub-frontend:build :datahub-web-react:build --parallel env: NODE_OPTIONS: "--max-old-space-size=3072" + - name: Gradle compile (jdk8) for legacy Spark + if: ${{ matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }} + run: | + ./gradlew -PjavaClassVersionDefault=8 :metadata-integration:java:spark-lineage:compileJava - uses: actions/upload-artifact@v3 if: always() with: diff --git a/build.gradle b/build.gradle index 27455f8592e6f..ba61d97f0ed6e 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,32 @@ buildscript { - ext.jdkVersion = 17 - ext.javaClassVersion = 11 + ext.jdkVersionDefault = 17 + ext.javaClassVersionDefault = 11 + + ext.jdkVersion = { p -> + // If Spring 6 is present, hard dependency on jdk17 + if (p.configurations.any { it.getDependencies().any{ + (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6.")) + || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test")) + }}) { + return 17 + } else { + // otherwise we can use the preferred default which can be overridden with a property: -PjdkVersionDefault + return p.hasProperty('jdkVersionDefault') ? Integer.valueOf((String) p.getProperty('jdkVersionDefault')) : ext.jdkVersionDefault + } + } + + ext.javaClassVersion = { p -> + // If Spring 6 is present, hard dependency on jdk17 + if (p.configurations.any { it.getDependencies().any{ + (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6.")) + || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test")) + }}) { + return 17 + } else { + // otherwise we can use the preferred default which can be overridden with a property: -PjavaClassVersionDefault + return p.hasProperty('javaClassVersionDefault') ? Integer.valueOf((String) p.getProperty('javaClassVersionDefault')) : ext.javaClassVersionDefault + } + } ext.junitJupiterVersion = '5.6.1' // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md @@ -217,6 +243,7 @@ project.ext.externalDependency = [ 'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion", 'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15', 'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46', + 'springBootAutoconfigureJdk11': 'org.springframework.boot:spring-boot-autoconfigure:2.7.18', 'testng': 'org.testng:testng:7.8.0', 'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion, 'testContainersJunit': 'org.testcontainers:junit-jupiter:' + testContainersVersion, @@ -252,23 +279,27 @@ allprojects { } } - if (project.plugins.hasPlugin('java') + /** + * If making changes to this section also see the sections for pegasus below + * which use project.plugins.hasPlugin('pegasus') + **/ + if (!project.plugins.hasPlugin('pegasus') && (project.plugins.hasPlugin('java') || project.plugins.hasPlugin('java-library') - || project.plugins.hasPlugin('application') - || project.plugins.hasPlugin('pegasus')) { + || project.plugins.hasPlugin('application'))) { java { toolchain { - languageVersion = JavaLanguageVersion.of(jdkVersion) + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) } } compileJava { - options.release = javaClassVersion + options.release = javaClassVersion(project) } + tasks.withType(JavaCompile).configureEach { javaCompiler = javaToolchains.compilerFor { - languageVersion = JavaLanguageVersion.of(jdkVersion) + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) } // Puts parameter names into compiled class files, necessary for Spring 6 options.compilerArgs.add("-parameters") @@ -276,24 +307,28 @@ allprojects { tasks.withType(JavaExec).configureEach { javaLauncher = javaToolchains.launcherFor { - languageVersion = JavaLanguageVersion.of(jdkVersion) + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) } } + } + + // not duplicated, need to set this outside and inside afterEvaluate + afterEvaluate { + /** + * If making changes to this section also see the sections for pegasus below + * which use project.plugins.hasPlugin('pegasus') + **/ + if (!project.plugins.hasPlugin('pegasus') && (project.plugins.hasPlugin('java') + || project.plugins.hasPlugin('java-library') + || project.plugins.hasPlugin('application'))) { - // not duplicated, need to set this outside and inside afterEvaluate - afterEvaluate { compileJava { - options.release = javaClassVersion - } - tasks.withType(JavaCompile).configureEach { - javaCompiler = javaToolchains.compilerFor { - languageVersion = JavaLanguageVersion.of(jdkVersion) - } + options.release = javaClassVersion(project) } tasks.withType(JavaExec).configureEach { javaLauncher = javaToolchains.launcherFor { - languageVersion = JavaLanguageVersion.of(jdkVersion) + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) } } } @@ -368,6 +403,30 @@ subprojects { dataTemplateCompile externalDependency.annotationApi // support > jdk8 restClientCompile spec.product.pegasus.restliClient } + + java { + toolchain { + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) + } + } + + compileJava { + options.release = javaClassVersion(project) + } + + tasks.withType(JavaCompile).configureEach { + javaCompiler = javaToolchains.compilerFor { + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) + } + // Puts parameter names into compiled class files, necessary for Spring 6 + options.compilerArgs.add("-parameters") + } + + tasks.withType(JavaExec).configureEach { + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) + } + } } afterEvaluate { @@ -394,6 +453,16 @@ subprojects { dataTemplateCompile externalDependency.annotationApi // support > jdk8 restClientCompile spec.product.pegasus.restliClient } + + compileJava { + options.release = javaClassVersion(project) + } + + tasks.withType(JavaExec).configureEach { + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) + } + } } } } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java index 453eddd3ae56c..806fd47c721ec 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java @@ -50,22 +50,21 @@ default boolean containsDuplicateAspects() { default Map> getUrnAspectsMap() { return getItems().stream() - .map(aspect -> Map.entry(aspect.getUrn().toString(), aspect.getAspectName())) + .map(aspect -> Pair.of(aspect.getUrn().toString(), aspect.getAspectName())) .collect( Collectors.groupingBy( - Map.Entry::getKey, Collectors.mapping(Map.Entry::getValue, Collectors.toSet()))); + Pair::getKey, Collectors.mapping(Pair::getValue, Collectors.toSet()))); } default Map> getNewUrnAspectsMap( Map> existingMap, List items) { Map> newItemsMap = items.stream() - .map(aspect -> Map.entry(aspect.getUrn().toString(), aspect.getAspectName())) + .map(aspect -> Pair.of(aspect.getUrn().toString(), aspect.getAspectName())) .collect( Collectors.groupingBy( - Map.Entry::getKey, - Collectors.mapping( - Map.Entry::getValue, Collectors.toCollection(HashSet::new)))); + Pair::getKey, + Collectors.mapping(Pair::getValue, Collectors.toCollection(HashSet::new)))); return newItemsMap.entrySet().stream() .filter( diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/GenericJsonPatch.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/GenericJsonPatch.java index c73ccbb2d93e3..484603b9c1f85 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/GenericJsonPatch.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/GenericJsonPatch.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.github.fge.jsonpatch.JsonPatch; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Map; import javax.annotation.Nonnull; @@ -24,7 +25,7 @@ public class GenericJsonPatch { @Nonnull public Map> getArrayPrimaryKeys() { - return arrayPrimaryKeys == null ? Map.of() : arrayPrimaryKeys; + return arrayPrimaryKeys == null ? Collections.emptyMap() : arrayPrimaryKeys; } @JsonIgnore diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginFactory.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginFactory.java index aec0a4cfa0706..5f35cb0447e48 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginFactory.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginFactory.java @@ -12,6 +12,7 @@ import io.github.classgraph.ClassInfo; import io.github.classgraph.MethodInfo; import io.github.classgraph.ScanResult; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -40,7 +41,7 @@ public static PluginFactory withCustomClasspath( } public static PluginFactory withConfig(@Nullable PluginConfiguration pluginConfiguration) { - return PluginFactory.withCustomClasspath(pluginConfiguration, List.of()); + return PluginFactory.withCustomClasspath(pluginConfiguration, Collections.emptyList()); } public static PluginFactory empty() { @@ -180,7 +181,7 @@ public EntityRegistryLoadResult.PluginLoadResult getPluginLoadResult() { private List buildAspectPayloadValidators( @Nullable PluginConfiguration pluginConfiguration) { return pluginConfiguration == null - ? List.of() + ? Collections.emptyList() : applyDisable( build( AspectPayloadValidator.class, @@ -190,7 +191,7 @@ private List buildAspectPayloadValidators( private List buildMutationHooks(@Nullable PluginConfiguration pluginConfiguration) { return pluginConfiguration == null - ? List.of() + ? Collections.emptyList() : applyDisable( build(MutationHook.class, pluginConfiguration.getMutationHooks(), HOOK_PACKAGES)); } @@ -198,7 +199,7 @@ private List buildMutationHooks(@Nullable PluginConfiguration plug private List buildMCLSideEffects( @Nullable PluginConfiguration pluginConfiguration) { return pluginConfiguration == null - ? List.of() + ? Collections.emptyList() : applyDisable( build(MCLSideEffect.class, pluginConfiguration.getMclSideEffects(), HOOK_PACKAGES)); } @@ -206,7 +207,7 @@ private List buildMCLSideEffects( private List buildMCPSideEffects( @Nullable PluginConfiguration pluginConfiguration) { return pluginConfiguration == null - ? List.of() + ? Collections.emptyList() : applyDisable( build(MCPSideEffect.class, pluginConfiguration.getMcpSideEffects(), HOOK_PACKAGES)); } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/config/PluginConfiguration.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/config/PluginConfiguration.java index a4d0678c130f3..a2caab7be5f80 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/config/PluginConfiguration.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/config/PluginConfiguration.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.aspect.plugins.config; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -11,10 +12,10 @@ @AllArgsConstructor @NoArgsConstructor public class PluginConfiguration { - private List aspectPayloadValidators = List.of(); - private List mutationHooks = List.of(); - private List mclSideEffects = List.of(); - private List mcpSideEffects = List.of(); + private List aspectPayloadValidators = Collections.emptyList(); + private List mutationHooks = Collections.emptyList(); + private List mclSideEffects = Collections.emptyList(); + private List mcpSideEffects = Collections.emptyList(); public static PluginConfiguration EMPTY = new PluginConfiguration(); diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectRetriever.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectRetriever.java index 00a20b3131c2a..11cd2352025ef 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectRetriever.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectRetriever.java @@ -1,10 +1,12 @@ package com.linkedin.metadata.aspect.plugins.validation; +import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; import com.linkedin.entity.Aspect; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.r2.RemoteInvocationException; import java.net.URISyntaxException; +import java.util.Collections; import java.util.Map; import java.util.Set; import javax.annotation.Nonnull; @@ -15,8 +17,8 @@ public interface AspectRetriever { @Nullable default Aspect getLatestAspectObject(@Nonnull final Urn urn, @Nonnull final String aspectName) throws RemoteInvocationException, URISyntaxException { - return getLatestAspectObjects(Set.of(urn), Set.of(aspectName)) - .getOrDefault(urn, Map.of()) + return getLatestAspectObjects(ImmutableSet.of(urn), ImmutableSet.of(aspectName)) + .getOrDefault(urn, Collections.emptyMap()) .get(aspectName); } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/ConfigEntityRegistry.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/ConfigEntityRegistry.java index bd9a6b6c9e589..41043995a3b77 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/ConfigEntityRegistry.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/ConfigEntityRegistry.java @@ -28,11 +28,13 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -67,7 +69,10 @@ public class ConfigEntityRegistry implements EntityRegistry { public ConfigEntityRegistry(Pair configFileClassPathPair) throws IOException { this( DataSchemaFactory.withCustomClasspath(configFileClassPathPair.getSecond()), - DataSchemaFactory.getClassLoader(configFileClassPathPair.getSecond()).stream().toList(), + DataSchemaFactory.getClassLoader(configFileClassPathPair.getSecond()) + .map(Stream::of) + .orElse(Stream.empty()) + .collect(Collectors.toList()), configFileClassPathPair.getFirst()); } @@ -112,7 +117,7 @@ private static Pair getFileAndClassPath(String entityRegistryRoot) } public ConfigEntityRegistry(InputStream configFileInputStream) { - this(DataSchemaFactory.getInstance(), List.of(), configFileInputStream); + this(DataSchemaFactory.getInstance(), Collections.emptyList(), configFileInputStream); } public ConfigEntityRegistry( diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PatchEntityRegistry.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PatchEntityRegistry.java index 35bfe935423f0..b82b905c50004 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PatchEntityRegistry.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PatchEntityRegistry.java @@ -32,6 +32,7 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -93,7 +94,10 @@ public PatchEntityRegistry( throws IOException, EntityRegistryException { this( DataSchemaFactory.withCustomClasspath(configFileClassPathPair.getSecond()), - DataSchemaFactory.getClassLoader(configFileClassPathPair.getSecond()).stream().toList(), + DataSchemaFactory.getClassLoader(configFileClassPathPair.getSecond()) + .map(Stream::of) + .orElse(Stream.empty()) + .collect(Collectors.toList()), configFileClassPathPair.getFirst(), registryName, registryVersion); diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/EntityRegistryLoadResult.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/EntityRegistryLoadResult.java index 076387909326b..12a29a7e1757a 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/EntityRegistryLoadResult.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/EntityRegistryLoadResult.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.models.registry.config; +import java.util.Collections; import java.util.Set; import lombok.Builder; import lombok.Data; @@ -23,9 +24,9 @@ public static class PluginLoadResult { private int mcpSideEffectCount; private int mclSideEffectCount; - @Builder.Default private Set validatorClasses = Set.of(); - @Builder.Default private Set mutationHookClasses = Set.of(); - @Builder.Default private Set mcpSideEffectClasses = Set.of(); - @Builder.Default private Set mclSideEffectClasses = Set.of(); + @Builder.Default private Set validatorClasses = Collections.emptySet(); + @Builder.Default private Set mutationHookClasses = Collections.emptySet(); + @Builder.Default private Set mcpSideEffectClasses = Collections.emptySet(); + @Builder.Default private Set mclSideEffectClasses = Collections.emptySet(); } } diff --git a/ingestion-scheduler/build.gradle b/ingestion-scheduler/build.gradle index dc9887406b8b4..9505ec57aa858 100644 --- a/ingestion-scheduler/build.gradle +++ b/ingestion-scheduler/build.gradle @@ -1,4 +1,6 @@ -apply plugin: 'java' +plugins { + id 'java' +} dependencies { implementation project(path: ':metadata-models') @@ -7,6 +9,7 @@ dependencies { implementation project(':metadata-service:configuration') implementation externalDependency.slf4jApi + implementation externalDependency.springContext compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok diff --git a/metadata-events/mxe-avro/build.gradle b/metadata-events/mxe-avro/build.gradle index 3aebc6bb1004d..58e82aff464d5 100644 --- a/metadata-events/mxe-avro/build.gradle +++ b/metadata-events/mxe-avro/build.gradle @@ -1,10 +1,12 @@ +plugins { + id 'java-library' + id 'io.acryl.gradle.plugin.avro' +} + configurations { avsc } -apply plugin: 'io.acryl.gradle.plugin.avro' -apply plugin: 'java-library' - dependencies { api externalDependency.avro implementation(externalDependency.avroCompiler) { diff --git a/metadata-events/mxe-registration/build.gradle b/metadata-events/mxe-registration/build.gradle index 2842dd935c7ee..d4b4d446996fa 100644 --- a/metadata-events/mxe-registration/build.gradle +++ b/metadata-events/mxe-registration/build.gradle @@ -1,4 +1,7 @@ -apply plugin: 'java' +plugins { + id 'java' + id 'pegasus' +} configurations { avroOriginal diff --git a/metadata-events/mxe-schemas/build.gradle b/metadata-events/mxe-schemas/build.gradle index 8dc8b71bd1cd8..ab0ea8b649e9d 100644 --- a/metadata-events/mxe-schemas/build.gradle +++ b/metadata-events/mxe-schemas/build.gradle @@ -1,5 +1,7 @@ -apply plugin: 'java-library' -apply plugin: 'pegasus' +plugins { + id 'java-library' + id 'pegasus' +} dependencies { dataModel project(path: ':li-utils', configuration: 'dataTemplate') diff --git a/metadata-events/mxe-utils-avro/build.gradle b/metadata-events/mxe-utils-avro/build.gradle index 98bfb9127b209..860ced6af2581 100644 --- a/metadata-events/mxe-utils-avro/build.gradle +++ b/metadata-events/mxe-utils-avro/build.gradle @@ -1,5 +1,6 @@ plugins { id 'java-library' + id 'pegasus' } dependencies { diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index 8e05b7ef8f5d6..873943fd43781 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -1,13 +1,13 @@ plugins { id("com.palantir.git-version") apply false + id 'java' + id 'com.github.johnrengelman.shadow' + id 'jacoco' + id 'signing' + id 'io.codearte.nexus-staging' + id 'maven-publish' } -apply plugin: 'java' -apply plugin: 'com.github.johnrengelman.shadow' -apply plugin: 'jacoco' -apply plugin: 'signing' -apply plugin: 'io.codearte.nexus-staging' -apply plugin: 'maven-publish' -apply plugin: 'org.hidetake.swagger.generator' + apply from: "../versioning.gradle" import org.apache.tools.ant.filters.ReplaceTokens @@ -28,10 +28,7 @@ dependencies { compileOnly externalDependency.httpAsyncClient implementation externalDependency.jacksonDataBind - implementation externalDependency.javaxValidation runtimeOnly externalDependency.jna - implementation externalDependency.springContext - implementation externalDependency.swaggerAnnotations implementation externalDependency.slf4jApi compileOnly externalDependency.lombok @@ -45,8 +42,6 @@ dependencies { testImplementation externalDependency.testContainers testImplementation externalDependency.httpAsyncClient testRuntimeOnly externalDependency.logbackClassic - - swaggerCodegen externalDependency.swaggerCli } task copyAvroSchemas { @@ -80,7 +75,7 @@ shadowJar { // preventing java multi-release JAR leakage // https://github.com/johnrengelman/shadow/issues/729 exclude('module-info.class', 'META-INF/versions/**', - '**/LICENSE', '**/LICENSE*.txt', '**/NOTICE', '**/NOTICE.txt', 'licenses/**', 'log4j2.xml', 'log4j.xml') + '**/LICENSE', '**/LICENSE*.txt', '**/NOTICE', '**/NOTICE.txt', 'licenses/**', 'log4j2.*', 'log4j.*') mergeServiceFiles() // we relocate namespaces manually, because we want to know exactly which libs we are exposing and why // we can move to automatic relocation using ConfigureShadowRelocation after we get to a good place on these first @@ -209,28 +204,4 @@ nexusStaging { //required only for projects registered in Sonatype after 2021-02-24 username = System.getenv("NEXUS_USERNAME") password = System.getenv("NEXUS_PASSWORD") -} - -tasks.register('generateOpenApiPojos', GenerateSwaggerCode) { - it.setInputFile(file("${project(':metadata-models').projectDir}/src/generatedJsonSchema/combined/open-api.yaml")) - it.setOutputDir(file("$projectDir/generated")) - it.setLanguage("spring") - it.setComponents(['models']) - it.setTemplateDir(file("$projectDir/src/main/resources/JavaSpring")) - it.setAdditionalProperties(["group-id" : "io.datahubproject", - "dateLibrary" : "java8", - "java8" : "true", - "modelPropertyNaming": "original", - "modelPackage" : "io.datahubproject.openapi.generated"] as Map) - - dependsOn ':metadata-models:generateJsonSchema' -} - -compileJava.dependsOn generateOpenApiPojos -processResources.dependsOn generateOpenApiPojos -sourceSets.main.java.srcDir "${generateOpenApiPojos.outputDir}/src/main/java" -sourceSets.main.resources.srcDir "${generateOpenApiPojos.outputDir}/src/main/resources" - -clean { - project.delete("$projectDir/generated") } \ No newline at end of file diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index 568b99acdf894..f96517d93fca6 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -1,5 +1,7 @@ -apply plugin: 'java-library' -apply plugin: 'org.hidetake.swagger.generator' +plugins { + id 'java-library' + id 'pegasus' +} configurations { enhance @@ -46,8 +48,8 @@ dependencies { implementation externalDependency.ebeanDdl implementation externalDependency.opentelemetryAnnotations implementation externalDependency.resilience4j - api externalDependency.springContext - implementation externalDependency.swaggerAnnotations + // Newer Spring libraries require JDK17 classes, allow for JDK11 + compileOnly externalDependency.springBootAutoconfigureJdk11 implementation(externalDependency.mixpanel) { exclude group: 'org.json', module: 'json' } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java index 0f22b75b69f10..bdc0332b040df 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java @@ -106,13 +106,16 @@ private AggregationBuilder facetToAggregationBuilder(final String inputFacet) { if (facet.contains(AGGREGATION_SPECIAL_TYPE_DELIMITER)) { List specialTypeFields = List.of(facet.split(AGGREGATION_SPECIAL_TYPE_DELIMITER)); switch (specialTypeFields.get(0)) { - case MISSING_SPECIAL_TYPE -> aggBuilder = - INDEX_VIRTUAL_FIELD.equalsIgnoreCase(specialTypeFields.get(1)) - ? AggregationBuilders.missing(inputFacet).field(getAggregationField("_index")) - : AggregationBuilders.missing(inputFacet) - .field(getAggregationField(specialTypeFields.get(1))); - default -> throw new UnsupportedOperationException( - "Unknown special type: " + specialTypeFields.get(0)); + case MISSING_SPECIAL_TYPE: + aggBuilder = + INDEX_VIRTUAL_FIELD.equalsIgnoreCase(specialTypeFields.get(1)) + ? AggregationBuilders.missing(inputFacet).field(getAggregationField("_index")) + : AggregationBuilders.missing(inputFacet) + .field(getAggregationField(specialTypeFields.get(1))); + break; + default: + throw new UnsupportedOperationException( + "Unknown special type: " + specialTypeFields.get(0)); } } else { aggBuilder = diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java index 05fa6f45fcb30..c5a5ade216bf7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java @@ -648,10 +648,12 @@ public static Map extractAggregationsFromResponse( if (aggregation == null) { return Collections.emptyMap(); } - if (aggregation instanceof ParsedTerms terms) { - return extractTermAggregations(terms, aggregationName.equals("_entityType")); - } else if (aggregation instanceof ParsedMissing missing) { - return Collections.singletonMap(missing.getName(), missing.getDocCount()); + if (aggregation instanceof ParsedTerms) { + return extractTermAggregations( + (ParsedTerms) aggregation, aggregationName.equals("_entityType")); + } else if (aggregation instanceof ParsedMissing) { + return Collections.singletonMap( + aggregation.getName(), ((ParsedMissing) aggregation).getDocCount()); } throw new UnsupportedOperationException( "Unsupported aggregation type: " + aggregation.getClass().getName()); @@ -669,10 +671,10 @@ private static Map recursivelyAddNestedSubAggs(@Nullable Aggregati if (aggs != null) { for (Map.Entry entry : aggs.getAsMap().entrySet()) { - if (entry.getValue() instanceof ParsedTerms terms) { - recurseTermsAgg(terms, aggResult, false); - } else if (entry.getValue() instanceof ParsedMissing missing) { - recurseMissingAgg(missing, aggResult); + if (entry.getValue() instanceof ParsedTerms) { + recurseTermsAgg((ParsedTerms) entry.getValue(), aggResult, false); + } else if (entry.getValue() instanceof ParsedMissing) { + recurseMissingAgg((ParsedMissing) entry.getValue(), aggResult); } else { throw new UnsupportedOperationException( "Unsupported aggregation type: " + entry.getValue().getClass().getName()); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java index 1f39a3947c47a..ee2d794471f6b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java @@ -129,7 +129,8 @@ public void handleChangeEvent(@Nonnull final MetadataChangeLog event) { .stream() .flatMap(mclSideEffect -> mclSideEffect.apply(List.of(batch), aspectRetriever)); - for (MCLBatchItem mclBatchItem : Stream.concat(Stream.of(batch), sideEffects).toList()) { + for (MCLBatchItem mclBatchItem : + Stream.concat(Stream.of(batch), sideEffects).collect(Collectors.toList())) { MetadataChangeLog hookEvent = mclBatchItem.getMetadataChangeLog(); if (UPDATE_CHANGE_TYPES.contains(hookEvent.getChangeType())) { handleUpdateChangeEvent(mclBatchItem); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java index 71ffd603c999f..a2b36b7d8ddb8 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java @@ -526,7 +526,7 @@ public TimeseriesScrollResult scrollAspects( List> resultPairs = Arrays.stream(response.getHits().getHits()) .map(ElasticSearchTimeseriesAspectService::toEnvAspectGenericDocument) - .toList(); + .collect(Collectors.toList()); return TimeseriesScrollResult.builder() .numResults(totalCount) diff --git a/metadata-jobs/common/build.gradle b/metadata-jobs/common/build.gradle index bdc3b7a44a98a..b0a3a6827b729 100644 --- a/metadata-jobs/common/build.gradle +++ b/metadata-jobs/common/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'java' + id 'java-library' } dependencies { diff --git a/metadata-models/build.gradle b/metadata-models/build.gradle index 86f404adb7fef..179e1eac177ac 100644 --- a/metadata-models/build.gradle +++ b/metadata-models/build.gradle @@ -18,10 +18,14 @@ dependencies { api project(path: ':li-utils', configuration: "dataTemplate") dataModel project(':li-utils') + // Newer Spring libraries require JDK17 classes, allow for JDK11 + compileOnly externalDependency.springBootAutoconfigureJdk11 + compileOnly externalDependency.annotationApi + compileOnly externalDependency.javaxValidation + compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok - compileOnly externalDependency.swaggerAnnotations - compileOnly externalDependency.springBootStarterValidation + api externalDependency.swaggerAnnotations compileOnly externalDependency.jacksonCore compileOnly externalDependency.jacksonDataBind diff --git a/metadata-models/src/main/resources/JavaSpring/model.mustache b/metadata-models/src/main/resources/JavaSpring/model.mustache index 72da42612777c..a048f249a6b3d 100644 --- a/metadata-models/src/main/resources/JavaSpring/model.mustache +++ b/metadata-models/src/main/resources/JavaSpring/model.mustache @@ -9,9 +9,9 @@ import java.io.Serializable; {{/serializableModel}} {{#useBeanValidation}} import org.springframework.validation.annotation.Validated; -import jakarta.validation.Valid; +import javax.validation.Valid; import com.fasterxml.jackson.annotation.JsonInclude; -import jakarta.validation.constraints.*; +import javax.validation.constraints.*; {{/useBeanValidation}} {{#jackson}} {{#withXml}} @@ -20,7 +20,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; {{/withXml}} {{/jackson}} {{#withXml}} -import jakarta.xml.bind.annotation.*; +import javax.xml.bind.annotation.*; {{/withXml}} {{/x-is-composed-model}} diff --git a/metadata-service/configuration/build.gradle b/metadata-service/configuration/build.gradle index 80cf6541261c2..f912e2ac01f0b 100644 --- a/metadata-service/configuration/build.gradle +++ b/metadata-service/configuration/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'java' + id 'java-library' } apply from: "../../gradle/versioning/versioning.gradle" @@ -7,8 +7,9 @@ dependencies { implementation externalDependency.jacksonDataBind implementation externalDependency.slf4jApi - implementation externalDependency.springCore - implementation externalDependency.springBeans + + // Newer Spring libraries require JDK17 classes, allow for JDK11 + compileOnly externalDependency.springBootAutoconfigureJdk11 compileOnly externalDependency.lombok diff --git a/metadata-service/restli-servlet-impl/build.gradle b/metadata-service/restli-servlet-impl/build.gradle index ec5b645ee233c..8d21bdd489505 100644 --- a/metadata-service/restli-servlet-impl/build.gradle +++ b/metadata-service/restli-servlet-impl/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'java' + id 'java-library' id 'pegasus' } diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntityRecommendationSource.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntityRecommendationSource.java index 546c2856c28ac..0a29ebfe46415 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntityRecommendationSource.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntityRecommendationSource.java @@ -8,6 +8,7 @@ import com.linkedin.metadata.recommendation.RecommendationParams; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -29,7 +30,7 @@ default Stream buildContent( entityUrns.stream() .map(UrnUtils::getUrn) .filter(urn -> getSupportedEntityTypes().contains(urn.getEntityType())) - .toList(); + .collect(Collectors.toList()); Set existingNonRemoved = entityService.exists(entities, false); return entities.stream().filter(existingNonRemoved::contains).map(this::buildContent); diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/service/RollbackService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/RollbackService.java index 22496b6c07806..666fe23a93187 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/service/RollbackService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/service/RollbackService.java @@ -126,7 +126,7 @@ public RollbackResponse rollbackIngestion( !row.getRunId().equals(runId) && !row.isKeyAspect() && !row.getAspectName().equals(Constants.STATUS_ASPECT_NAME)) - .toList(); + .collect(Collectors.toList()); long unsafeEntitiesCount = affectedAspectsList.stream() @@ -212,7 +212,7 @@ public RollbackResponse rollbackIngestion( !row.getRunId().equals(runId) && !row.isKeyAspect() && !row.getAspectName().equals(Constants.STATUS_ASPECT_NAME)) - .toList(); + .collect(Collectors.toList()); long affectedAspects = affectedAspectsList.size(); long unsafeEntitiesCount = diff --git a/metadata-utils/build.gradle b/metadata-utils/build.gradle index 3d65675219624..919d93c5f9fe1 100644 --- a/metadata-utils/build.gradle +++ b/metadata-utils/build.gradle @@ -1,5 +1,6 @@ plugins { id 'java-library' + id 'pegasus' } dependencies {