Merge branch 'master' into docs/add-markdown-link-check

datahub-project · Sep 8, 2023 · b8f00e6 · b8f00e6
2 parents 1fe3652 + 68ae3bf
commit b8f00e6
Show file tree

Hide file tree

Showing 300 changed files with 11,859 additions and 5,948 deletions.
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -45,6 +45,7 @@ jobs:
       unique_slim_tag: ${{ steps.tag.outputs.unique_slim_tag }}
       unique_full_tag: ${{ steps.tag.outputs.unique_full_tag }}
       publish: ${{ steps.publish.outputs.publish }}
+      python_release_version: ${{ steps.tag.outputs.python_release_version }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -58,6 +59,7 @@ jobs:
           echo "unique_tag=$(get_unique_tag)" >> $GITHUB_OUTPUT
           echo "unique_slim_tag=$(get_unique_tag)-slim" >> $GITHUB_OUTPUT
           echo "unique_full_tag=$(get_unique_tag)-full" >> $GITHUB_OUTPUT
+          echo "python_release_version=$(get_python_docker_release_v)" >> $GITHUB_OUTPUT
       - name: Check whether publishing enabled
         id: publish
         env:
@@ -573,7 +575,7 @@ jobs:
         with:
           image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head' }}
       - name: Build and push Slim Image
-        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' }}
+        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
         uses: ./.github/actions/docker-custom-build-and-push
         with:
           target: final
@@ -582,6 +584,7 @@ jobs:
           build-args: |
             BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
             DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head' }}
+            RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }}
             APP_ENV=slim
           tags: ${{ needs.setup.outputs.slim_tag }}
           username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
@@ -655,7 +658,7 @@ jobs:
         with:
           image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_full_tag || 'head' }}
       - name: Build and push Full Image
-        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' }}
+        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
         uses: ./.github/actions/docker-custom-build-and-push
         with:
           target: final
@@ -664,6 +667,7 @@ jobs:
           build-args: |
             BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
             DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_full_tag || 'head' }}
+            RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }}
           tags: ${{ needs.setup.outputs.unique_full_tag }}
           username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
           password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
@@ -673,7 +677,7 @@ jobs:
           platforms: linux/amd64,linux/arm64/v8
       - name: Compute Tag (Full)
         id: tag
-        run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_full_tag || 'head' }}" >> $GITHUB_OUTPUT
+        run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_full_tag || 'head' }}" >> $GITHUB_OUTPUT
   datahub_ingestion_full_scan:
     permissions:
       contents: read # for actions/checkout to fetch code

diff --git a/.github/workflows/spark-smoke-test.yml b/.github/workflows/spark-smoke-test.yml
@@ -30,6 +30,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 800
+          fetch-tags: true
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:
@@ -40,6 +43,8 @@ jobs:
           python-version: "3.7"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
+      - name: Remove images
+        run: docker image prune -a -f || true
       - name: Smoke test
         run: |
           ./gradlew :metadata-integration:java:spark-lineage:integrationTest \

diff --git a/build.gradle b/build.gradle
@@ -1,7 +1,7 @@
 buildscript {
   ext.junitJupiterVersion = '5.6.1'
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
-  ext.pegasusVersion = '29.22.16'
+  ext.pegasusVersion = '29.45.0'
   ext.mavenVersion = '3.6.3'
   ext.springVersion = '5.3.29'
   ext.springBootVersion = '2.7.14'
@@ -18,6 +18,7 @@ buildscript {
   ext.hadoop3Version = '3.3.5'
   ext.kafkaVersion = '2.3.0'
   ext.hazelcastVersion = '5.3.1'
+  ext.ebeanVersion = '12.16.1'
 
   ext.docker_registry = 'linkedin'
 
@@ -29,16 +30,16 @@ buildscript {
     classpath 'io.acryl.gradle.plugin:gradle-avro-plugin:0.8.1'
     classpath 'org.springframework.boot:spring-boot-gradle-plugin:' + springBootVersion
     classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0"
-    classpath "com.palantir.gradle.gitversion:gradle-git-version:0.12.3"
-    classpath "org.gradle.playframework:gradle-playframework:0.12"
-    classpath "gradle.plugin.org.hidetake:gradle-swagger-generator-plugin:2.18.1"
+    classpath "com.palantir.gradle.gitversion:gradle-git-version:3.0.0"
+    classpath "org.gradle.playframework:gradle-playframework:0.14"
+    classpath "gradle.plugin.org.hidetake:gradle-swagger-generator-plugin:2.19.1"
   }
 }
 
 plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.0-rc2'
   id 'com.github.johnrengelman.shadow' version '6.1.0'
-  id "com.palantir.docker" version "0.35.0"
+  id 'com.palantir.docker' version '0.35.0'
   // https://blog.ltgt.net/javax-jakarta-mess-and-gradle-solution/
   // TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"
 }
@@ -86,16 +87,17 @@ project.ext.externalDependency = [
     'dgraph4j' : 'io.dgraph:dgraph4j:21.03.1',
     'dropwizardMetricsCore': 'io.dropwizard.metrics:metrics-core:4.2.3',
     'dropwizardMetricsJmx': 'io.dropwizard.metrics:metrics-jmx:4.2.3',
-    'ebean': 'io.ebean:ebean:11.33.3',
-    'ebeanAgent': 'io.ebean:ebean-agent:11.27.1',
+    'ebean': 'io.ebean:ebean:' + ebeanVersion,
+    'ebeanAgent': 'io.ebean:ebean-agent:' + ebeanVersion,
+    'ebeanDdl': 'io.ebean:ebean-ddl-generator:' + ebeanVersion,
     'elasticSearchRest': 'org.elasticsearch.client:elasticsearch-rest-high-level-client:' + elasticsearchVersion,
     'elasticSearchTransport': 'org.elasticsearch.client:transport:' + elasticsearchVersion,
     'findbugsAnnotations': 'com.google.code.findbugs:annotations:3.0.1',
     'graphqlJava': 'com.graphql-java:graphql-java:19.5',
     'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:19.1',
     'gson': 'com.google.code.gson:gson:2.8.9',
     'guice': 'com.google.inject:guice:4.2.3',
-    'guava': 'com.google.guava:guava:27.0.1-jre',
+    'guava': 'com.google.guava:guava:32.1.2-jre',
     'h2': 'com.h2database:h2:2.1.214',
     'hadoopCommon':'org.apache.hadoop:hadoop-common:2.7.2',
     'hadoopMapreduceClient':'org.apache.hadoop:hadoop-mapreduce-client-core:2.7.2',
@@ -144,7 +146,7 @@ project.ext.externalDependency = [
     'log4jApi': "org.apache.logging.log4j:log4j-api:$log4jVersion",
     'log4j12Api': "org.slf4j:log4j-over-slf4j:$slf4jVersion",
     'log4j2Api': "org.apache.logging.log4j:log4j-to-slf4j:$log4jVersion",
-    'lombok': 'org.projectlombok:lombok:1.18.12',
+    'lombok': 'org.projectlombok:lombok:1.18.16',
     'mariadbConnector': 'org.mariadb.jdbc:mariadb-java-client:2.6.0',
     'mavenArtifact': "org.apache.maven:maven-artifact:$mavenVersion",
     'mixpanel': 'com.mixpanel:mixpanel-java:1.4.4',
@@ -185,7 +187,7 @@ project.ext.externalDependency = [
     'springBeans': "org.springframework:spring-beans:$springVersion",
     'springContext': "org.springframework:spring-context:$springVersion",
     'springCore': "org.springframework:spring-core:$springVersion",
-    'springDocUI': 'org.springdoc:springdoc-openapi-ui:1.6.7',
+    'springDocUI': 'org.springdoc:springdoc-openapi-ui:1.6.14',
     'springJdbc': "org.springframework:spring-jdbc:$springVersion",
     'springWeb': "org.springframework:spring-web:$springVersion",
     'springWebMVC': "org.springframework:spring-webmvc:$springVersion",
@@ -195,9 +197,11 @@ project.ext.externalDependency = [
     'springBootStarterWeb': "org.springframework.boot:spring-boot-starter-web:$springBootVersion",
     'springBootStarterJetty': "org.springframework.boot:spring-boot-starter-jetty:$springBootVersion",
     'springBootStarterCache': "org.springframework.boot:spring-boot-starter-cache:$springBootVersion",
+    'springBootStarterValidation': "org.springframework.boot:spring-boot-starter-validation:$springBootVersion",
     'springKafka': 'org.springframework.kafka:spring-kafka:2.8.11',
     'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
     'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.1.12',
+    'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.41',
     'testng': 'org.testng:testng:7.3.0',
     'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion,
     'testContainersJunit': 'org.testcontainers:junit-jupiter:' + testContainersVersion,
@@ -239,7 +243,7 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
 
 subprojects {
 
-  apply plugin: 'maven'
+  apply plugin: 'maven-publish'
   apply plugin: 'com.gorylenko.gradle-git-properties'
 
   gitProperties {
@@ -253,7 +257,7 @@ subprojects {
 
   plugins.withType(JavaPlugin) {
     dependencies {
-      testCompile externalDependency.testng
+      testImplementation externalDependency.testng
       constraints {
         implementation('io.netty:netty-all:4.1.86.Final')
         implementation('org.apache.commons:commons-compress:1.21')

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -5,12 +5,15 @@ buildscript {
 }
 
 dependencies {
-  compile('io.acryl:json-schema-avro:0.1.5') {
+  implementation('io.acryl:json-schema-avro:0.1.5') {
     exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
     exclude group: 'com.google.guava', module: 'guava'
   }
-  compile 'com.google.guava:guava:27.0.1-jre'
-  compile 'com.fasterxml.jackson.core:jackson-databind:2.13.5'
-  compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.5'
-  compile 'commons-io:commons-io:2.11.0'
+  implementation 'com.google.guava:guava:32.1.2-jre'
+  implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.5'
+  implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.5'
+  implementation 'commons-io:commons-io:2.11.0'
+
+  compileOnly 'org.projectlombok:lombok:1.18.14'
+  annotationProcessor 'org.projectlombok:lombok:1.18.14'
 }
diff --git a/buildSrc/src/main/java/com/linkedin/metadata/models/registry/config b/buildSrc/src/main/java/com/linkedin/metadata/models/registry/config
@@ -0,0 +1 @@
+../../../../../../../../../entity-registry/src/main/java/com/linkedin/metadata/models/registry/config
diff --git a/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java b/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java
@@ -20,11 +20,9 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.stream.Collectors;
+
 import org.gradle.api.DefaultTask;
-import org.gradle.api.tasks.CacheableTask;
-import org.gradle.api.tasks.InputDirectory;
-import org.gradle.api.tasks.OutputDirectory;
-import org.gradle.api.tasks.TaskAction;
+import org.gradle.api.tasks.*;
 
 import static com.github.fge.processing.ProcessingUtil.*;
 import static org.apache.commons.io.FilenameUtils.*;
@@ -34,18 +32,33 @@
 public class GenerateJsonSchemaTask extends DefaultTask {
   private String inputDirectory;
   private String outputDirectory;
+
   private ArrayNode aspectType;
   private Path combinedDirectory;
+
   private Path jsonDirectory;
   public static final String sep = FileSystems.getDefault().getSeparator();
 
   private static final JsonNodeFactory NODE_FACTORY = JacksonUtils.nodeFactory();
 
+  private static final OpenApiEntities openApiEntities = new OpenApiEntities(NODE_FACTORY);
+
+  @InputFile
+  @PathSensitive(PathSensitivity.NAME_ONLY)
+  public String getEntityRegistryYaml() {
+    return openApiEntities.getEntityRegistryYaml();
+  }
+
+  public void setEntityRegistryYaml(String entityRegistryYaml) {
+    openApiEntities.setEntityRegistryYaml(entityRegistryYaml);
+  }
+
   public void setInputDirectory(String inputDirectory) {
     this.inputDirectory = inputDirectory;
   }
 
   @InputDirectory
+  @PathSensitive(PathSensitivity.NAME_ONLY)
   public String getInputDirectory() {
    return inputDirectory;
   }
@@ -80,6 +93,7 @@ public void generate() throws IOException {
         .filter(Files::isRegularFile)
         .map(Path::toFile)
         .forEach(this::generateSchema);
+
     List<ObjectNode> nodesList = Files.walk(jsonDirectory)
         .filter(Files::isRegularFile)
         .filter(path -> {
@@ -110,6 +124,18 @@ public void generate() throws IOException {
       }
       schemasNode.setAll(definitions);
     });
+
+    combinedDirectory = Paths.get(outputDirectory + sep + "combined");
+    try {
+      Files.createDirectory(combinedDirectory);
+    } catch (FileAlreadyExistsException fae) {
+      // No-op
+    }
+
+    // Add additional components and paths
+    openApiEntities.setCombinedDirectory(combinedDirectory);
+    ObjectNode extendedNode = openApiEntities.entityExtension(nodesList, schemasNode);
+
     /*
     Minimal OpenAPI header
     openapi: 3.0.1
@@ -133,29 +159,23 @@ public void generate() throws IOException {
         .set("paths", NODE_FACTORY.objectNode()
             .set("/path", NODE_FACTORY.objectNode()
                 .set("get", NODE_FACTORY.objectNode().set("tags", NODE_FACTORY.arrayNode().add("path")))));
-    JsonNode combinedSchemaDefinitionsYaml = ((ObjectNode) NODE_FACTORY.objectNode().set("components",
-        NODE_FACTORY.objectNode().set("schemas", schemasNode))).setAll(yamlHeader);
+
+    JsonNode combinedSchemaDefinitionsYaml = extendedNode.setAll(yamlHeader);
 
     final String yaml = new YAMLMapper().writeValueAsString(combinedSchemaDefinitionsYaml)
-        .replaceAll("definitions", "components/schemas")
-        .replaceAll("\n\\s+- type: \"null\"", "");
+            .replaceAll("definitions", "components/schemas")
+            .replaceAll("\n\\s+description: null", "")
+            .replaceAll("\n\\s+- type: \"null\"", "");
 
-    combinedDirectory = Paths.get(outputDirectory + sep + "combined");
-    try {
-      Files.createDirectory(combinedDirectory);
-    } catch (FileAlreadyExistsException fae) {
-      // No-op
-    }
     Files.write(Paths.get(combinedDirectory + sep + "open-api.yaml"),
         yaml.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE,
         StandardOpenOption.TRUNCATE_EXISTING);
 
-    JsonNode combinedSchemaDefinitionsJson = NODE_FACTORY.objectNode().set("definitions",schemasNode);
+    JsonNode combinedSchemaDefinitionsJson = NODE_FACTORY.objectNode().set("definitions", extendedNode);
     String prettySchema = JacksonUtils.prettyPrint(combinedSchemaDefinitionsJson);
     Files.write(Paths.get(Paths.get(outputDirectory) + sep + "combined" + sep + "schema.json"),
         prettySchema.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE,
         StandardOpenOption.TRUNCATE_EXISTING);
-
   }
 
   private final HashSet<String> filenames = new HashSet<>();
@@ -185,5 +205,4 @@ private void generateSchema(final File file) {
       throw new RuntimeException(e);
     }
   }
-
 }