feat(openapi): entity endpoints & analytics raw (#8537)

datahub-project · Sep 7, 2023 · 56c00f5 · 56c00f5
1 parent cf16684
commit 56c00f5
Show file tree

Hide file tree

Showing 58 changed files with 3,309 additions and 209 deletions.
diff --git a/build.gradle b/build.gradle
@@ -146,7 +146,7 @@ project.ext.externalDependency = [
     'log4jApi': "org.apache.logging.log4j:log4j-api:$log4jVersion",
     'log4j12Api': "org.slf4j:log4j-over-slf4j:$slf4jVersion",
     'log4j2Api': "org.apache.logging.log4j:log4j-to-slf4j:$log4jVersion",
-    'lombok': 'org.projectlombok:lombok:1.18.12',
+    'lombok': 'org.projectlombok:lombok:1.18.16',
     'mariadbConnector': 'org.mariadb.jdbc:mariadb-java-client:2.6.0',
     'mavenArtifact': "org.apache.maven:maven-artifact:$mavenVersion",
     'mixpanel': 'com.mixpanel:mixpanel-java:1.4.4',
@@ -187,7 +187,7 @@ project.ext.externalDependency = [
     'springBeans': "org.springframework:spring-beans:$springVersion",
     'springContext': "org.springframework:spring-context:$springVersion",
     'springCore': "org.springframework:spring-core:$springVersion",
-    'springDocUI': 'org.springdoc:springdoc-openapi-ui:1.6.7',
+    'springDocUI': 'org.springdoc:springdoc-openapi-ui:1.6.14',
     'springJdbc': "org.springframework:spring-jdbc:$springVersion",
     'springWeb': "org.springframework:spring-web:$springVersion",
     'springWebMVC': "org.springframework:spring-webmvc:$springVersion",
@@ -197,9 +197,11 @@ project.ext.externalDependency = [
     'springBootStarterWeb': "org.springframework.boot:spring-boot-starter-web:$springBootVersion",
     'springBootStarterJetty': "org.springframework.boot:spring-boot-starter-jetty:$springBootVersion",
     'springBootStarterCache': "org.springframework.boot:spring-boot-starter-cache:$springBootVersion",
+    'springBootStarterValidation': "org.springframework.boot:spring-boot-starter-validation:$springBootVersion",
     'springKafka': 'org.springframework.kafka:spring-kafka:2.8.11',
     'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
     'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.1.12',
+    'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.41',
     'testng': 'org.testng:testng:7.3.0',
     'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion,
     'testContainersJunit': 'org.testcontainers:junit-jupiter:' + testContainersVersion,

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -13,4 +13,7 @@ dependencies {
   implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.5'
   implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.5'
   implementation 'commons-io:commons-io:2.11.0'
+
+  compileOnly 'org.projectlombok:lombok:1.18.14'
+  annotationProcessor 'org.projectlombok:lombok:1.18.14'
 }
diff --git a/buildSrc/src/main/java/com/linkedin/metadata/models/registry/config b/buildSrc/src/main/java/com/linkedin/metadata/models/registry/config
@@ -0,0 +1 @@
+../../../../../../../../../entity-registry/src/main/java/com/linkedin/metadata/models/registry/config
diff --git a/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java b/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java
@@ -20,6 +20,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.stream.Collectors;
+
 import org.gradle.api.DefaultTask;
 import org.gradle.api.tasks.*;
 
@@ -31,13 +32,27 @@
 public class GenerateJsonSchemaTask extends DefaultTask {
   private String inputDirectory;
   private String outputDirectory;
+
   private ArrayNode aspectType;
   private Path combinedDirectory;
+
   private Path jsonDirectory;
   public static final String sep = FileSystems.getDefault().getSeparator();
 
   private static final JsonNodeFactory NODE_FACTORY = JacksonUtils.nodeFactory();
 
+  private static final OpenApiEntities openApiEntities = new OpenApiEntities(NODE_FACTORY);
+
+  @InputFile
+  @PathSensitive(PathSensitivity.NAME_ONLY)
+  public String getEntityRegistryYaml() {
+    return openApiEntities.getEntityRegistryYaml();
+  }
+
+  public void setEntityRegistryYaml(String entityRegistryYaml) {
+    openApiEntities.setEntityRegistryYaml(entityRegistryYaml);
+  }
+
   public void setInputDirectory(String inputDirectory) {
     this.inputDirectory = inputDirectory;
   }
@@ -78,6 +93,7 @@ public void generate() throws IOException {
         .filter(Files::isRegularFile)
         .map(Path::toFile)
         .forEach(this::generateSchema);
+
     List<ObjectNode> nodesList = Files.walk(jsonDirectory)
         .filter(Files::isRegularFile)
         .filter(path -> {
@@ -108,6 +124,18 @@ public void generate() throws IOException {
       }
       schemasNode.setAll(definitions);
     });
+
+    combinedDirectory = Paths.get(outputDirectory + sep + "combined");
+    try {
+      Files.createDirectory(combinedDirectory);
+    } catch (FileAlreadyExistsException fae) {
+      // No-op
+    }
+
+    // Add additional components and paths
+    openApiEntities.setCombinedDirectory(combinedDirectory);
+    ObjectNode extendedNode = openApiEntities.entityExtension(nodesList, schemasNode);
+
     /*
     Minimal OpenAPI header
     openapi: 3.0.1
@@ -131,29 +159,23 @@ public void generate() throws IOException {
         .set("paths", NODE_FACTORY.objectNode()
             .set("/path", NODE_FACTORY.objectNode()
                 .set("get", NODE_FACTORY.objectNode().set("tags", NODE_FACTORY.arrayNode().add("path")))));
-    JsonNode combinedSchemaDefinitionsYaml = ((ObjectNode) NODE_FACTORY.objectNode().set("components",
-        NODE_FACTORY.objectNode().set("schemas", schemasNode))).setAll(yamlHeader);
+
+    JsonNode combinedSchemaDefinitionsYaml = extendedNode.setAll(yamlHeader);
 
     final String yaml = new YAMLMapper().writeValueAsString(combinedSchemaDefinitionsYaml)
-        .replaceAll("definitions", "components/schemas")
-        .replaceAll("\n\\s+- type: \"null\"", "");
+            .replaceAll("definitions", "components/schemas")
+            .replaceAll("\n\\s+description: null", "")
+            .replaceAll("\n\\s+- type: \"null\"", "");
 
-    combinedDirectory = Paths.get(outputDirectory + sep + "combined");
-    try {
-      Files.createDirectory(combinedDirectory);
-    } catch (FileAlreadyExistsException fae) {
-      // No-op
-    }
     Files.write(Paths.get(combinedDirectory + sep + "open-api.yaml"),
         yaml.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE,
         StandardOpenOption.TRUNCATE_EXISTING);
 
-    JsonNode combinedSchemaDefinitionsJson = NODE_FACTORY.objectNode().set("definitions",schemasNode);
+    JsonNode combinedSchemaDefinitionsJson = NODE_FACTORY.objectNode().set("definitions", extendedNode);
     String prettySchema = JacksonUtils.prettyPrint(combinedSchemaDefinitionsJson);
     Files.write(Paths.get(Paths.get(outputDirectory) + sep + "combined" + sep + "schema.json"),
         prettySchema.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE,
         StandardOpenOption.TRUNCATE_EXISTING);
-
   }
 
   private final HashSet<String> filenames = new HashSet<>();
@@ -183,5 +205,4 @@ private void generateSchema(final File file) {
       throw new RuntimeException(e);
     }
   }
-
 }