diff --git a/.gitignore b/.gitignore index e4da3b21047af..6c340de3e64db 100644 --- a/.gitignore +++ b/.gitignore @@ -11,8 +11,11 @@ *gma-create-all.sql *gma-drop-all.sql -# Pegasus & Avro +# Pegasus, Avro, & other schemas **/src/mainGenerated* +**/src/generatedJsonSchema +**/metadata-io/generated +**/metadata-integration/java/datahub-client/generated **/src/testGenerated* metadata-events/mxe-registration/src/main/resources/**/*.avsc diff --git a/build.gradle b/build.gradle index 0d61e9aad82d2..af341ee52c7aa 100644 --- a/build.gradle +++ b/build.gradle @@ -16,6 +16,7 @@ buildscript { } classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0" classpath "com.palantir.gradle.gitversion:gradle-git-version:0.12.3" + classpath "gradle.plugin.org.hidetake:gradle-swagger-generator-plugin:2.18.1" } } @@ -80,10 +81,12 @@ project.ext.externalDependency = [ 'jacksonDataFormatYaml': 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.10', 'javatuples': 'org.javatuples:javatuples:1.2', 'javaxInject' : 'javax.inject:javax.inject:1', + 'javaxValidation' : 'javax.validation:validation-api:2.0.1.Final', 'jerseyCore': 'org.glassfish.jersey.core:jersey-client:2.25.1', 'jerseyGuava': 'org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1', 'jettyJaas': 'org.eclipse.jetty:jetty-jaas:9.4.32.v20200930', 'jgrapht': 'org.jgrapht:jgrapht-core:1.5.1', + 'jsonSchemaAvro': 'com.github.fge:json-schema-avro:0.1.4', 'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1', 'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion", 'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion", diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle new file mode 100644 index 0000000000000..2d94fd876c31f --- /dev/null +++ b/buildSrc/build.gradle @@ -0,0 +1,16 @@ +apply plugin: 'java' + +buildscript { + apply from: '../repositories.gradle' +} + +dependencies { + compile('io.acryl:json-schema-avro:0.1.5') { + exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind' + exclude group: 'com.google.guava', module: 'guava' + } + compile 'com.google.guava:guava:27.0.1-jre' + compile 'com.fasterxml.jackson.core:jackson-databind:2.9.10.7' + compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.8.11' + compile 'commons-io:commons-io:2.11.0' +} \ No newline at end of file diff --git a/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java b/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java new file mode 100644 index 0000000000000..a5a843d91b1eb --- /dev/null +++ b/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java @@ -0,0 +1,189 @@ +package io.datahubproject; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; +import com.github.fge.jackson.JacksonUtils; +import com.github.fge.jackson.JsonLoader; +import com.github.fge.jsonschema.core.exceptions.ProcessingException; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; +import org.gradle.api.DefaultTask; +import org.gradle.api.tasks.CacheableTask; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.TaskAction; + +import static com.github.fge.processing.ProcessingUtil.*; +import static org.apache.commons.io.FilenameUtils.*; + + +@CacheableTask +public class GenerateJsonSchemaTask extends DefaultTask { + private String inputDirectory; + private String outputDirectory; + private ArrayNode aspectType; + private Path combinedDirectory; + private Path jsonDirectory; + public static final String sep = FileSystems.getDefault().getSeparator(); + + private static final JsonNodeFactory NODE_FACTORY = JacksonUtils.nodeFactory(); + + public void setInputDirectory(String inputDirectory) { + this.inputDirectory = inputDirectory; + } + + @InputDirectory + public String getInputDirectory() { + return inputDirectory; + } + + @OutputDirectory + public String getOutputDirectory() { + return outputDirectory; + } + + public void setOutputDirectory(String outputDirectory) { + this.outputDirectory = outputDirectory; + } + + @TaskAction + public void generate() throws IOException { + Path baseDir = Paths.get(inputDirectory); + aspectType = NODE_FACTORY.arrayNode(); + + Path schemaDirectory = Paths.get(outputDirectory); + jsonDirectory = Paths.get(schemaDirectory + sep + "json"); + try { + Files.createDirectory(schemaDirectory); + } catch (FileAlreadyExistsException fae) { + // No-op + } + try { + Files.createDirectory(jsonDirectory); + } catch (FileAlreadyExistsException fae) { + // No-op + } + Files.walk(baseDir) + .filter(Files::isRegularFile) + .map(Path::toFile) + .forEach(this::generateSchema); + List nodesList = Files.walk(jsonDirectory) + .filter(Files::isRegularFile) + .filter(path -> { + String fileName = path.toFile().getName(); + return !getBaseName(fileName).contains("ChangeEvent") && !getBaseName(fileName).contains("AuditEvent"); + }) + .map(Path::toFile) + .map(file -> { + try { + return (ObjectNode) JsonLoader.fromFile(file); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + + ObjectNode schemasNode = NODE_FACTORY.objectNode(); + nodesList.forEach(objectNode -> { + ObjectNode definitions = (ObjectNode) objectNode.get("definitions"); + // Modify GenericAspect and EnvelopedAspect to have all Aspect subtypes + if (definitions.has("GenericAspect")) { + ObjectNode genericAspect = (ObjectNode) definitions.get("GenericAspect"); + ((ObjectNode) genericAspect.get("properties")).replace("value", NODE_FACTORY.objectNode().set("oneOf", aspectType)); + } + if (definitions.has("EnvelopedAspect")) { + ObjectNode envelopedAspect = (ObjectNode) definitions.get("EnvelopedAspect"); + ((ObjectNode) envelopedAspect.get("properties")).replace("value", NODE_FACTORY.objectNode().set("oneOf", aspectType)); + } + schemasNode.setAll(definitions); + }); + /* + Minimal OpenAPI header + openapi: 3.0.1 + info: + title: OpenAPI definition + version: v0 + servers: + - url: http://localhost:8080/openapi + description: Generated server url + paths: + /path: + get: + tags: + - path + */ + ObjectNode yamlHeader = (ObjectNode) ((ObjectNode) NODE_FACTORY.objectNode() + .put("openapi", "3.0.1") + .set("info", NODE_FACTORY.objectNode() + .put("title", "OpenAPI Definition") + .put("version", "v0"))) + .set("paths", NODE_FACTORY.objectNode() + .set("/path", NODE_FACTORY.objectNode() + .set("get", NODE_FACTORY.objectNode().set("tags", NODE_FACTORY.arrayNode().add("path"))))); + JsonNode combinedSchemaDefinitionsYaml = ((ObjectNode) NODE_FACTORY.objectNode().set("components", + NODE_FACTORY.objectNode().set("schemas", schemasNode))).setAll(yamlHeader); + + final String yaml = new YAMLMapper().writeValueAsString(combinedSchemaDefinitionsYaml) + .replaceAll("definitions", "components/schemas") + .replaceAll("\n\\s+- type: \"null\"", ""); + + combinedDirectory = Paths.get(outputDirectory + sep + "combined"); + try { + Files.createDirectory(combinedDirectory); + } catch (FileAlreadyExistsException fae) { + // No-op + } + Files.write(Paths.get(combinedDirectory + sep + "open-api.yaml"), + yaml.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); + + JsonNode combinedSchemaDefinitionsJson = NODE_FACTORY.objectNode().set("definitions",schemasNode); + String prettySchema = JacksonUtils.prettyPrint(combinedSchemaDefinitionsJson); + Files.write(Paths.get(Paths.get(outputDirectory) + sep + "combined" + sep + "schema.json"), + prettySchema.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); + + } + + private final HashSet filenames = new HashSet<>(); + private void generateSchema(final File file) { + final String fileBaseName; + try { + final JsonNode schema = JsonLoader.fromFile(file); + final JsonNode result = buildResult(schema.toString()); + String prettySchema = JacksonUtils.prettyPrint(result); + Path absolutePath = file.getAbsoluteFile().toPath(); + if (absolutePath.endsWith(Paths.get("com", "linkedin", "metadata", "aspect", "EnvelopedAspect.avsc"))) { + fileBaseName = "EnvelopedTimeseriesAspect"; + prettySchema = prettySchema.replaceAll("EnvelopedAspect", "EnvelopedTimeseriesAspect"); + } else if (!filenames.add(file.getName())) { + System.out.println("Not processing legacy schema " + absolutePath); + return; + } else { + fileBaseName = getBaseName(file.getName()); + } + Files.write(Paths.get(jsonDirectory + sep + fileBaseName + ".json"), + prettySchema.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); + if (schema.has("Aspect")) { + aspectType.add(NODE_FACTORY.objectNode().put("$ref", "#/definitions/" + getBaseName(file.getName()))); + } + } catch (IOException | ProcessingException e) { + throw new RuntimeException(e); + } + } + +} \ No newline at end of file diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 3e810bfcaea97..860a977c3237d 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -203,6 +203,13 @@ module.exports = { ], }, ], + OpenAPI: [ + { + label: "Usage Guide", + type: "doc", + id: "docs/api/openapi/openapi-usage-guide", + }, + ], "Usage Guides": [ "docs/policies", "docs/domains", diff --git a/docs/api/openapi/openapi-usage-guide.md b/docs/api/openapi/openapi-usage-guide.md new file mode 100644 index 0000000000000..b13c7363c7991 --- /dev/null +++ b/docs/api/openapi/openapi-usage-guide.md @@ -0,0 +1,503 @@ +# Getting Started + +## Introduction to OpenAPI - What And Why + +The OpenAPI standard is a widely used documentation and design approach for APIs. Historically, we have published our RESTful APIs +under the Rest.li standard that is used internally by LinkedIn that we inherited from the beginning of the project. + +Rest.li is a very opinionated framework that has not seen wide adoption in the Open Source community, so users are often unfamiliar +with the best ways to interact with these endpoints. To make it easier to integrate with DataHub, we are publishing an OpenAPI based set of endpoints. + +## Using DataHub's OpenAPI - Where and How + +Currently, the OpenAPI endpoints are isolated to a servlet on GMS and are automatically deployed with a GMS server. +The servlet includes auto-generation of an OpenAPI UI, also known as Swagger, which is available at `:/openapi/swagger-ui/index.html`. +This is also exposed through DataHub frontend as a proxy with the same endpoint, but GMS host and port replaced with DataHub frontend's url +and is available in the top right dropdown under the user profile picture as a link. Note that it is possible to get +the raw json or yaml formats of the OpenAPI spec by navigating to `/openapi/v3/api-docs` or `/openapi/v3/api-docs.yaml`. +The raw forms can be fed into codegen systems to generate client side code in the language of your choice that support OpenAPI format. We have noticed varying +degrees of maturity with different languages in these codegen systems so some may require customizations to be fully compatible. + +The OpenAPI UI includes explorable schemas for request and response objects that are fully documented. The models used +in the OpenAPI UI are all autogenerated at build time from the PDL models to JSON Schema compatible Java Models. + +Programmatic usage of the models can be done through the Java Rest Emitter which includes the generated models. A minimal +Java project for emitting to the OpenAPI endpoints would need the following dependencies (gradle format): + +```groovy +dependencies { + implementation 'io.acryl:datahub-client:' + implementation 'org.apache.httpcomponents:httpclient:' + implementation 'org.apache.httpcomponents:httpasyncclient:' +} +``` + +and would construct a list of `UpsertAspectRequest`s to emit: + +```java +import io.datahubproject.openapi.generated.DatasetProperties; +import datahub.client.rest.RestEmitter; +import datahub.event.UpsertAspectRequest; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; + + +public class Main { + public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { + RestEmitter emitter = RestEmitter.createWithDefaults(); + + List requests = new ArrayList<>(); + UpsertAspectRequest upsertAspectRequest = UpsertAspectRequest.builder() + .entityType("dataset") + .entityUrn("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project.my-other-dataset.user-table,PROD)") + .aspect(new DatasetProperties().description("This is the canonical User profile dataset")) + .build(); + UpsertAspectRequest upsertAspectRequest2 = UpsertAspectRequest.builder() + .entityType("dataset") + .entityUrn("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project.another-dataset.user-table,PROD)") + .aspect(new DatasetProperties().description("This is the canonical User profile dataset 2")) + .build(); + requests.add(upsertAspectRequest); + requests.add(upsertAspectRequest2); + System.out.println(emitter.emit(requests, null).get()); + System.exit(0); + } +} +``` + +### Example requests + +#### Curl + +##### POST + +```shell +curl --location --request POST 'localhost:8080/openapi/entities/v1/' \ +--header 'Content-Type: application/json' \ +--header 'Accept: application/json' \ +--header 'Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.eyJhY3RvclR5cGUiOiJVU0VSIiwiYWN0b3JJZCI6ImRhdGFodWIiLCJ0eXBlIjoiUEVSU09OQUwiLCJ2ZXJzaW9uIjoiMSIsImV4cCI6MTY1MDY2MDY1NSwianRpIjoiM2E4ZDY3ZTItOTM5Yi00NTY3LWE0MjYtZDdlMDA1ZGU3NjJjIiwic3ViIjoiZGF0YWh1YiIsImlzcyI6ImRhdGFodWItbWV0YWRhdGEtc2VydmljZSJ9.pp_vW2u1tiiTT7U0nDF2EQdcayOMB8jatiOA8Je4JJA' \ +--data-raw '[ + { + "aspect": { + "__type": "SchemaMetadata", + "schemaName": "SampleHdfsSchema", + "platform": "urn:li:dataPlatform:platform", + "platformSchema": { + "__type": "MySqlDDL", + "tableSchema": "schema" + }, + "version": 0, + "created": { + "time": 1621882982738, + "actor": "urn:li:corpuser:etl", + "impersonator": "urn:li:corpuser:jdoe" + }, + "lastModified": { + "time": 1621882982738, + "actor": "urn:li:corpuser:etl", + "impersonator": "urn:li:corpuser:jdoe" + }, + "hash": "", + "fields": [ + { + "fieldPath": "county_fips_codefg", + "jsonPath": "null", + "nullable": true, + "description": "null", + "type": { + "type": { + "__type": "StringType" + } + }, + "nativeDataType": "String()", + "recursive": false + }, + { + "fieldPath": "county_name", + "jsonPath": "null", + "nullable": true, + "description": "null", + "type": { + "type": { + "__type": "StringType" + } + }, + "nativeDataType": "String()", + "recursive": false + } + ] + }, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)" + } +]' +``` + +##### GET + +```shell +curl --location --request GET 'localhost:8080/openapi/entities/v1/latest?urns=urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)&aspectNames=schemaMetadata' \ +--header 'Accept: application/json' \ +--header 'Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.eyJhY3RvclR5cGUiOiJVU0VSIiwiYWN0b3JJZCI6ImRhdGFodWIiLCJ0eXBlIjoiUEVSU09OQUwiLCJ2ZXJzaW9uIjoiMSIsImV4cCI6MTY1MDY2MDY1NSwianRpIjoiM2E4ZDY3ZTItOTM5Yi00NTY3LWE0MjYtZDdlMDA1ZGU3NjJjIiwic3ViIjoiZGF0YWh1YiIsImlzcyI6ImRhdGFodWItbWV0YWRhdGEtc2VydmljZSJ9.pp_vW2u1tiiTT7U0nDF2EQdcayOMB8jatiOA8Je4JJA' +``` + +##### DELETE + +```shell +curl --location --request DELETE 'localhost:8080/openapi/entities/v1/?urns=urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)&soft=true' \ +--header 'Accept: application/json' \ +--header 'Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.eyJhY3RvclR5cGUiOiJVU0VSIiwiYWN0b3JJZCI6ImRhdGFodWIiLCJ0eXBlIjoiUEVSU09OQUwiLCJ2ZXJzaW9uIjoiMSIsImV4cCI6MTY1MDY2MDY1NSwianRpIjoiM2E4ZDY3ZTItOTM5Yi00NTY3LWE0MjYtZDdlMDA1ZGU3NjJjIiwic3ViIjoiZGF0YWh1YiIsImlzcyI6ImRhdGFodWItbWV0YWRhdGEtc2VydmljZSJ9.pp_vW2u1tiiTT7U0nDF2EQdcayOMB8jatiOA8Je4JJA' +``` + +#### Postman Collection + +Collection includes a POST, GET, and DELETE for a single entity with a SchemaMetadata aspect + +```json +{ + "info": { + "_postman_id": "87b7401c-a5dc-47e4-90b4-90fe876d6c28", + "name": "DataHub OpenAPI", + "description": "A description", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "item": [ + { + "name": "entities/v1", + "item": [ + { + "name": "post Entities 1", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Accept", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "[\n {\n \"aspect\": {\n \"__type\": \"SchemaMetadata\",\n \"schemaName\": \"SampleHdfsSchema\",\n \"platform\": \"urn:li:dataPlatform:platform\",\n \"platformSchema\": {\n \"__type\": \"MySqlDDL\",\n \"tableSchema\": \"schema\"\n },\n \"version\": 0,\n \"created\": {\n \"time\": 1621882982738,\n \"actor\": \"urn:li:corpuser:etl\",\n \"impersonator\": \"urn:li:corpuser:jdoe\"\n },\n \"lastModified\": {\n \"time\": 1621882982738,\n \"actor\": \"urn:li:corpuser:etl\",\n \"impersonator\": \"urn:li:corpuser:jdoe\"\n },\n \"hash\": \"\",\n \"fields\": [\n {\n \"fieldPath\": \"county_fips_codefg\",\n \"jsonPath\": \"null\",\n \"nullable\": true,\n \"description\": \"null\",\n \"type\": {\n \"type\": {\n \"__type\": \"StringType\"\n }\n },\n \"nativeDataType\": \"String()\",\n \"recursive\": false\n },\n {\n \"fieldPath\": \"county_name\",\n \"jsonPath\": \"null\",\n \"nullable\": true,\n \"description\": \"null\",\n \"type\": {\n \"type\": {\n \"__type\": \"StringType\"\n }\n },\n \"nativeDataType\": \"String()\",\n \"recursive\": false\n }\n ]\n },\n \"aspectName\": \"schemaMetadata\",\n \"entityType\": \"dataset\",\n \"entityUrn\": \"urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)\"\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/openapi/entities/v1/", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openapi", + "entities", + "v1", + "" + ] + } + }, + "response": [ + { + "name": "OK", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"aspect\": {\n \"value\": \"\"\n },\n \"aspectName\": \"aliquip ipsum tempor\",\n \"entityType\": \"ut est\",\n \"entityUrn\": \"enim in nulla\",\n \"entityKeyAspect\": {\n \"value\": \"\"\n }\n },\n {\n \"aspect\": {\n \"value\": \"\"\n },\n \"aspectName\": \"ipsum id\",\n \"entityType\": \"deser\",\n \"entityUrn\": \"aliqua sit\",\n \"entityKeyAspect\": {\n \"value\": \"\"\n }\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/entities/v1/", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "entities", + "v1", + "" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "cookie": [], + "body": "[\n \"c\",\n \"labore dolor exercitation in\"\n]" + } + ] + }, + { + "name": "delete Entities", + "request": { + "method": "DELETE", + "header": [ + { + "key": "Accept", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/openapi/entities/v1/?urns=urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)&soft=true", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openapi", + "entities", + "v1", + "" + ], + "query": [ + { + "key": "urns", + "value": "urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)", + "description": "(Required) A list of raw urn strings, only supports a single entity type per request." + }, + { + "key": "urns", + "value": "labore dolor exercitation in", + "description": "(Required) A list of raw urn strings, only supports a single entity type per request.", + "disabled": true + }, + { + "key": "soft", + "value": "true", + "description": "Determines whether the delete will be soft or hard, defaults to true for soft delete" + } + ] + } + }, + "response": [ + { + "name": "OK", + "originalRequest": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{baseUrl}}/entities/v1/?urns=urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)&soft=true", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "entities", + "v1", + "" + ], + "query": [ + { + "key": "urns", + "value": "urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)" + }, + { + "key": "urns", + "value": "officia occaecat elit dolor", + "disabled": true + }, + { + "key": "soft", + "value": "true" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "cookie": [], + "body": "[\n {\n \"rowsRolledBack\": [\n {\n \"urn\": \"urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)\"\n }\n ],\n \"rowsDeletedFromEntityDeletion\": 1\n }\n]" + } + ] + }, + { + "name": "get Entities", + "protocolProfileBehavior": { + "disableUrlEncoding": false + }, + "request": { + "method": "GET", + "header": [ + { + "key": "Accept", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/openapi/entities/v1/latest?urns=urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)&aspectNames=schemaMetadata", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openapi", + "entities", + "v1", + "latest" + ], + "query": [ + { + "key": "urns", + "value": "urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)", + "description": "(Required) A list of raw urn strings, only supports a single entity type per request." + }, + { + "key": "urns", + "value": "labore dolor exercitation in", + "description": "(Required) A list of raw urn strings, only supports a single entity type per request.", + "disabled": true + }, + { + "key": "aspectNames", + "value": "schemaMetadata", + "description": "The list of aspect names to retrieve" + }, + { + "key": "aspectNames", + "value": "labore dolor exercitation in", + "description": "The list of aspect names to retrieve", + "disabled": true + } + ] + } + }, + "response": [ + { + "name": "OK", + "originalRequest": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/entities/v1/latest?urns=urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)&aspectNames=schemaMetadata", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "entities", + "v1", + "latest" + ], + "query": [ + { + "key": "urns", + "value": "non exercitation occaecat", + "disabled": true + }, + { + "key": "urns", + "value": "urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)" + }, + { + "key": "aspectNames", + "value": "non exercitation occaecat", + "disabled": true + }, + { + "key": "aspectNames", + "value": "schemaMetadata" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "cookie": [], + "body": "{\n \"responses\": {\n \"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)\": {\n \"entityName\": \"dataset\",\n \"urn\": \"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)\",\n \"aspects\": {\n \"datasetKey\": {\n \"name\": \"datasetKey\",\n \"type\": \"VERSIONED\",\n \"version\": 0,\n \"value\": {\n \"__type\": \"DatasetKey\",\n \"platform\": \"urn:li:dataPlatform:hive\",\n \"name\": \"SampleHiveDataset\",\n \"origin\": \"PROD\"\n },\n \"created\": {\n \"time\": 1650657843351,\n \"actor\": \"urn:li:corpuser:__datahub_system\"\n }\n },\n \"schemaMetadata\": {\n \"name\": \"schemaMetadata\",\n \"type\": \"VERSIONED\",\n \"version\": 0,\n \"value\": {\n \"__type\": \"SchemaMetadata\",\n \"schemaName\": \"SampleHiveSchema\",\n \"platform\": \"urn:li:dataPlatform:hive\",\n \"version\": 0,\n \"created\": {\n \"time\": 1581407189000,\n \"actor\": \"urn:li:corpuser:jdoe\"\n },\n \"lastModified\": {\n \"time\": 1581407189000,\n \"actor\": \"urn:li:corpuser:jdoe\"\n },\n \"hash\": \"\",\n \"platformSchema\": {\n \"__type\": \"KafkaSchema\",\n \"documentSchema\": \"{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"SampleHiveSchema\\\",\\\"namespace\\\":\\\"com.linkedin.dataset\\\",\\\"doc\\\":\\\"Sample Hive dataset\\\",\\\"fields\\\":[{\\\"name\\\":\\\"field_foo\\\",\\\"type\\\":[\\\"string\\\"]},{\\\"name\\\":\\\"field_bar\\\",\\\"type\\\":[\\\"boolean\\\"]}]}\"\n },\n \"fields\": [\n {\n \"fieldPath\": \"field_foo\",\n \"nullable\": false,\n \"description\": \"Foo field description\",\n \"type\": {\n \"type\": {\n \"__type\": \"BooleanType\"\n }\n },\n \"nativeDataType\": \"varchar(100)\",\n \"recursive\": false,\n \"isPartOfKey\": true\n },\n {\n \"fieldPath\": \"field_bar\",\n \"nullable\": false,\n \"description\": \"Bar field description\",\n \"type\": {\n \"type\": {\n \"__type\": \"BooleanType\"\n }\n },\n \"nativeDataType\": \"boolean\",\n \"recursive\": false,\n \"isPartOfKey\": false\n }\n ]\n },\n \"created\": {\n \"time\": 1650610810000,\n \"actor\": \"urn:li:corpuser:UNKNOWN\"\n }\n }\n }\n }\n }\n}" + } + ] + } + ], + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{token}}", + "type": "string" + } + ] + }, + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ], + "variable": [ + { + "key": "baseUrl", + "value": "localhost:8080", + "type": "string" + }, + { + "key": "token", + "value": "eyJhbGciOiJIUzI1NiJ9.eyJhY3RvclR5cGUiOiJVU0VSIiwiYWN0b3JJZCI6ImRhdGFodWIiLCJ0eXBlIjoiUEVSU09OQUwiLCJ2ZXJzaW9uIjoiMSIsImV4cCI6MTY1MDY2MDY1NSwianRpIjoiM2E4ZDY3ZTItOTM5Yi00NTY3LWE0MjYtZDdlMDA1ZGU3NjJjIiwic3ViIjoiZGF0YWh1YiIsImlzcyI6ImRhdGFodWItbWV0YWRhdGEtc2VydmljZSJ9.pp_vW2u1tiiTT7U0nDF2EQdcayOMB8jatiOA8Je4JJA", + "type": "default" + } + ] +} +``` \ No newline at end of file diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index bd5db85fdacc0..a6edfca073d79 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -7,6 +7,7 @@ apply plugin: 'jacoco' apply plugin: 'signing' apply plugin: 'io.codearte.nexus-staging' apply plugin: 'maven-publish' +apply plugin: 'org.hidetake.swagger.generator' import org.apache.tools.ant.filters.ReplaceTokens jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation @@ -16,12 +17,17 @@ dependencies { implementation project(':metadata-models') shadow externalDependency.httpAsyncClient // we want our clients to provide this implementation externalDependency.jacksonDataBind + implementation externalDependency.javaxValidation + implementation externalDependency.springContext + implementation externalDependency.swaggerAnnotations compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok testCompile externalDependency.httpAsyncClient // needed as shadow excludes it testCompile externalDependency.mockito testCompile externalDependency.mockServer testCompile externalDependency.mockServerClient + + swaggerCodegen 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.33' } jacocoTestReport { @@ -188,7 +194,6 @@ publishing { } } - signing { def signingKey = findProperty("signingKey") def signingPassword = System.getenv("SIGNING_PASSWORD") @@ -201,3 +206,29 @@ nexusStaging { username = System.getenv("NEXUS_USERNAME") password = System.getenv("NEXUS_PASSWORD") } + +tasks.register('generateOpenApiPojos', GenerateSwaggerCode) { + it.setInputFile( + file( + "${project(':metadata-models').projectDir}/src/generatedJsonSchema/combined/open-api.yaml" + ) + ) + it.setOutputDir(file("$projectDir/generated")) + it.setLanguage("spring") + it.setComponents(['models']) + it.setTemplateDir(file("$projectDir/src/main/resources/JavaSpring")) + it.setAdditionalProperties([ + "group-id" : "io.datahubproject", + "dateLibrary" : "java8", + "java8" : "true", + "modelPropertyNaming" : "original", + "modelPackage" : "io.datahubproject.openapi.generated"] as Map) + + dependsOn ':metadata-models:generateJsonSchema' +} + +compileJava.dependsOn generateOpenApiPojos +sourceSets.main.java.srcDir "${generateOpenApiPojos.outputDir}/src/main/java" +sourceSets.main.resources.srcDir "${generateOpenApiPojos.outputDir}/src/main/resources" + +checkstyleMain.exclude '**/generated/**' diff --git a/metadata-integration/java/datahub-client/scripts/check_jar.sh b/metadata-integration/java/datahub-client/scripts/check_jar.sh index 69a2c25805749..f47e9a13aa8a5 100755 --- a/metadata-integration/java/datahub-client/scripts/check_jar.sh +++ b/metadata-integration/java/datahub-client/scripts/check_jar.sh @@ -11,7 +11,17 @@ jar -tvf $jarFile |\ grep -v "pegasus/" |\ grep -v "legacyPegasusSchemas/" |\ grep -v " com/$" |\ + grep -v " org/$" |\ + grep -v " io/$" |\ grep -v "git.properties" |\ + grep -v "org/springframework" |\ + grep -v "org/aopalliance" |\ + grep -v "javax/" |\ + grep -v "io/swagger" |\ + grep -v "JavaSpring" |\ + grep -v "java-header-style.xml" |\ + grep -v "xml-header-style.xml" |\ + grep -v "license.header" |\ grep -v "client.properties" if [ $? -ne 0 ]; then diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/Emitter.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/Emitter.java index 4b6a4ad3dfebb..25bcba5f7d4c6 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/Emitter.java +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/Emitter.java @@ -2,8 +2,10 @@ import com.linkedin.mxe.MetadataChangeProposal; import datahub.event.MetadataChangeProposalWrapper; +import datahub.event.UpsertAspectRequest; import java.io.Closeable; import java.io.IOException; +import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import javax.annotation.Nonnull; @@ -70,4 +72,13 @@ default Future emit(@Nonnull MetadataChangeProposal mcp) */ boolean testConnection() throws IOException, ExecutionException, InterruptedException; + /** + * Asynchronously emit a {@link UpsertAspectRequest}. + * @param request request with with metadata aspect to upsert into DataHub + * @return a {@link Future} for callers to inspect the result of the operation or block until one is available + * @throws IOException + */ + Future emit(List request, Callback callback) + throws IOException; + } diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java index c5ea552c54bb7..54fc313ec0245 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java @@ -11,9 +11,11 @@ import datahub.client.MetadataWriteResponse; import datahub.event.EventFormatter; import datahub.event.MetadataChangeProposalWrapper; +import datahub.event.UpsertAspectRequest; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -58,6 +60,7 @@ public class RestEmitter implements Emitter { private final RestEmitterConfig config; private final String ingestProposalUrl; + private final String ingestOpenApiUrl; private final String configUrl; private final ObjectMapper objectMapper = new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL); @@ -82,6 +85,7 @@ public RestEmitter(RestEmitterConfig config) { this.httpClient = this.config.getAsyncHttpClientBuilder().build(); this.httpClient.start(); this.ingestProposalUrl = this.config.getServer() + "/aspects?action=ingestProposal"; + this.ingestOpenApiUrl = config.getServer() + "/openapi/entities/v1/"; this.configUrl = this.config.getServer() + "/config"; this.eventFormatter = this.config.getEventFormatter(); } @@ -240,4 +244,69 @@ public boolean testConnection() throws IOException, ExecutionException, Interrup public void close() throws IOException { this.httpClient.close(); } + + @Override + public Future emit(List request, Callback callback) + throws IOException { + log.debug("Emit: URL: {}, Payload: {}\n", this.ingestOpenApiUrl, request); + return this.postOpenAPI(request, callback); + } + + private Future postOpenAPI(List payload, Callback callback) + throws IOException { + HttpPost httpPost = new HttpPost(ingestOpenApiUrl); + httpPost.setHeader("Content-Type", "application/json"); + httpPost.setHeader("Accept", "application/json"); + this.config.getExtraHeaders().forEach((k, v) -> httpPost.setHeader(k, v)); + if (this.config.getToken() != null) { + httpPost.setHeader("Authorization", "Bearer " + this.config.getToken()); + } + httpPost.setEntity(new StringEntity(objectMapper.writeValueAsString(payload))); + AtomicReference responseAtomicReference = new AtomicReference<>(); + CountDownLatch responseLatch = new CountDownLatch(1); + FutureCallback httpCallback = new FutureCallback() { + @Override + public void completed(HttpResponse response) { + MetadataWriteResponse writeResponse = null; + try { + writeResponse = mapResponse(response); + responseAtomicReference.set(writeResponse); + } catch (Exception e) { + // do nothing + } + responseLatch.countDown(); + if (callback != null) { + try { + callback.onCompletion(writeResponse); + } catch (Exception e) { + log.error("Error executing user callback on completion.", e); + } + } + } + + @Override + public void failed(Exception ex) { + if (callback != null) { + try { + callback.onFailure(ex); + } catch (Exception e) { + log.error("Error executing user callback on failure.", e); + } + } + } + + @Override + public void cancelled() { + if (callback != null) { + try { + callback.onFailure(new RuntimeException("Cancelled")); + } catch (Exception e) { + log.error("Error executing user callback on failure due to cancellation.", e); + } + } + } + }; + Future requestFuture = httpClient.execute(httpPost, httpCallback); + return new MetadataResponseFuture(requestFuture, responseAtomicReference, responseLatch); + } } \ No newline at end of file diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/event/UpsertAspectRequest.java b/metadata-integration/java/datahub-client/src/main/java/datahub/event/UpsertAspectRequest.java new file mode 100644 index 0000000000000..eb834ccea2b91 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/event/UpsertAspectRequest.java @@ -0,0 +1,39 @@ +package datahub.event; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import io.datahubproject.openapi.generated.OneOfGenericAspectValue; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Builder; +import lombok.Value; + + +@JsonInclude(JsonInclude.Include.NON_NULL) +@Value +@Builder +@JsonDeserialize(builder = UpsertAspectRequest.UpsertAspectRequestBuilder.class) +public class UpsertAspectRequest { + + @JsonProperty("entityType") + @Schema(required = true, description = "The name of the entity matching with its definition in the entity registry") + String entityType; + + @JsonProperty("entityUrn") + @Schema(description = "Urn of the entity to be updated with the corresponding aspect, required if entityKey is null") + String entityUrn; + + @JsonProperty("entityKeyAspect") + @Schema(description = "A key aspect referencing the entity to be updated, required if entityUrn is null") + OneOfGenericAspectValue entityKeyAspect; + + @JsonProperty("aspect") + @Schema(required = true, description = "Aspect value to be upserted") + OneOfGenericAspectValue aspect; + + @JsonPOJOBuilder(withPrefix = "") + public static class UpsertAspectRequestBuilder { + + } +} diff --git a/metadata-integration/java/datahub-client/src/main/resources/JavaSpring/interface.mustache b/metadata-integration/java/datahub-client/src/main/resources/JavaSpring/interface.mustache new file mode 100644 index 0000000000000..ae2faa2ce0b49 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/resources/JavaSpring/interface.mustache @@ -0,0 +1,21 @@ +{{#jackson}} +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +{{/jackson}} +/** +* {{#description}}{{.}}{{/description}}{{^description}}{{classname}}{{/description}} +*/ +{{#jackson}} +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.EXISTING_PROPERTY, + property = "__type") +@JsonSubTypes({ + {{#subTypes}} + @JsonSubTypes.Type(value = {{classname}}.class, name = "{{classname}}"){{^@last}},{{/@last}} + {{/subTypes}} +}) +{{/jackson}} +public interface {{{classname}}} { + +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-client/src/main/resources/JavaSpring/model.mustache b/metadata-integration/java/datahub-client/src/main/resources/JavaSpring/model.mustache new file mode 100644 index 0000000000000..a048f249a6b3d --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/resources/JavaSpring/model.mustache @@ -0,0 +1,41 @@ +package {{package}}; + +{{^x-is-composed-model}} +import java.util.Objects; +{{#imports}}import {{import}}; +{{/imports}} +{{#serializableModel}} +import java.io.Serializable; +{{/serializableModel}} +{{#useBeanValidation}} +import org.springframework.validation.annotation.Validated; +import javax.validation.Valid; +import com.fasterxml.jackson.annotation.JsonInclude; +import javax.validation.constraints.*; +{{/useBeanValidation}} +{{#jackson}} +{{#withXml}} +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +{{/withXml}} +{{/jackson}} +{{#withXml}} +import javax.xml.bind.annotation.*; +{{/withXml}} +{{/x-is-composed-model}} + +{{#models}} +{{#model}} +{{#isComposedModel}} +{{>interface}} +{{/isComposedModel}} +{{^isComposedModel}} +{{#isEnum}} +{{>enumOuterClass}} +{{/isEnum}} +{{^isEnum}} +{{>pojo}} +{{/isEnum}} +{{/isComposedModel}} +{{/model}} +{{/models}} \ No newline at end of file diff --git a/metadata-integration/java/datahub-client/src/main/resources/JavaSpring/pojo.mustache b/metadata-integration/java/datahub-client/src/main/resources/JavaSpring/pojo.mustache new file mode 100644 index 0000000000000..8747765dcb031 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/resources/JavaSpring/pojo.mustache @@ -0,0 +1,166 @@ +{{#if hasVars}} +{{else}} +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +{{/if}} +/** + * {{#description}}{{.}}{{/description}}{{^description}}{{classname}}{{/description}} + */{{#description}} +{{#useOas2}}@ApiModel{{/useOas2}}{{^useOas2}}@Schema{{/useOas2}}(description = "{{{description}}}"){{/description}} +{{#useBeanValidation}}@Validated{{/useBeanValidation}} +{{>generatedAnnotation}}{{#discriminator}}{{>typeInfoAnnotation}}{{/discriminator}}{{>xmlAnnotation}} +@JsonInclude(JsonInclude.Include.NON_NULL) +public class {{classname}} {{#parent}}extends {{{parent}}}{{/parent}} {{#serializableModel}}implements Serializable {{#interfaceModels}}, {{classname}}{{^@last}}, {{/@last}}{{#@last}} {{/@last}}{{/interfaceModels}}{{/serializableModel}}{{^serializableModel}}{{#interfaceModels}}{{#@first}}implements {{/@first}}{{classname}}{{^@last}}, {{/@last}}{{#@last}}{{/@last}}{{/interfaceModels}}{{/serializableModel}} { +{{#serializableModel}} + private static final long serialVersionUID = 1L; + +{{/serializableModel}} +{{#if interfaceModels}} + + @JsonProperty(value = "__type", defaultValue = "{{classname}}") + private String __type = "{{classname}}"; + + /** + * Name of this subclass in SimpleClassName format + * @return __type + **/ + @Schema(required = true, description = "Name of this subclass in SimpleClassName format", allowableValues = {"{{classname}}"}, + defaultValue = "{{classname}}") + @NotNull + public String get__type() { + return __type; + } +{{/if}} + + {{#vars}} + {{#baseItems this}} + {{#isEnum}} +{{>enumClass}} + {{/isEnum}} + {{/baseItems}} + {{#jackson}} + {{#vendorExtensions.x-is-discriminator-property}} + @JsonTypeId + {{/vendorExtensions.x-is-discriminator-property}} + {{^vendorExtensions.x-is-discriminator-property}} + @JsonProperty("{{baseName}}"){{#withXml}} + @JacksonXmlProperty({{#isXmlAttribute}}isAttribute = true, {{/isXmlAttribute}}{{#xmlNamespace}}namespace="{{xmlNamespace}}", {{/xmlNamespace}}localName = "{{#xmlName}}{{xmlName}}{{/xmlName}}{{^xmlName}}{{baseName}}{{/xmlName}}"){{/withXml}} + {{/vendorExtensions.x-is-discriminator-property}} + {{/jackson}} + {{#gson}} + @SerializedName("{{baseName}}") + {{/gson}} + {{#isContainer}} + {{#useBeanValidation}}@Valid{{/useBeanValidation}} + private {{{datatypeWithEnum}}} {{name}}{{#required}} = {{{defaultValue}}}{{/required}}{{^required}} = null{{/required}}; + {{/isContainer}} + {{^isContainer}} + private {{{datatypeWithEnum}}} {{name}} = {{{defaultValue}}}; + {{/isContainer}} + + {{/vars}} + {{#vars}} + public {{classname}} {{name}}({{{datatypeWithEnum}}} {{name}}) { + this.{{name}} = {{name}}; + return this; + } + {{#isListContainer}} + + public {{classname}} add{{nameInCamelCase}}Item({{{items.datatypeWithEnum}}} {{name}}Item) { + {{^required}} + if (this.{{name}} == null) { + this.{{name}} = {{{defaultValue}}}; + } + {{/required}} + this.{{name}}.add({{name}}Item); + return this; + } + {{/isListContainer}} + {{#isMapContainer}} + + public {{classname}} put{{nameInCamelCase}}Item(String key, {{{items.datatypeWithEnum}}} {{name}}Item) { + {{^required}} + if (this.{{name}} == null) { + this.{{name}} = {{{defaultValue}}}; + } + {{/required}} + this.{{name}}.put(key, {{name}}Item); + return this; + } + {{/isMapContainer}} + + /** + {{#description}} + * {{{description}}} + {{/description}} + {{^description}} + * Get {{name}} + {{/description}} + {{#minimum}} + * minimum: {{minimum}} + {{/minimum}} + {{#maximum}} + * maximum: {{maximum}} + {{/maximum}} + * @return {{name}} + **/ + {{#vendorExtensions.extraAnnotation}} + {{{vendorExtensions.extraAnnotation}}} + {{/vendorExtensions.extraAnnotation}} + {{#useOas2}} + @ApiModelProperty({{#example}}example = "{{{example}}}", {{/example}}{{#required}}required = {{required}}, {{/required}}{{#isReadOnly}}readOnly = {{{isReadOnly}}}, {{/isReadOnly}}value = "{{{description}}}") + {{/useOas2}} + {{^useOas2}} + @Schema({{#example}}example = "{{{example}}}", {{/example}}{{#required}}required = {{required}}, {{/required}}{{#isReadOnly}}accessMode = Schema.AccessMode.READ_ONLY, {{/isReadOnly}}description = "{{{description}}}") + {{/useOas2}} + {{#useBeanValidation}}{{>beanValidation}}{{/useBeanValidation}} public {{{datatypeWithEnum}}} {{getter}}() { + return {{name}}; + } + + public void {{setter}}({{{datatypeWithEnum}}} {{name}}) { + this.{{name}} = {{name}}; + } + + {{/vars}} + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + }{{#hasVars}} + {{classname}} {{classVarName}} = ({{classname}}) o; + return {{#vars}}Objects.equals(this.{{name}}, {{classVarName}}.{{name}}){{#hasMore}} && + {{/hasMore}}{{/vars}}{{#parent}} && + super.equals(o){{/parent}};{{/hasVars}}{{^hasVars}} + return true;{{/hasVars}} + } + + @Override + public int hashCode() { + return Objects.hash({{#vars}}{{name}}{{#hasMore}}, {{/hasMore}}{{/vars}}{{#parent}}{{#hasVars}}, {{/hasVars}}super.hashCode(){{/parent}}); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class {{classname}} {\n"); + {{#parent}}sb.append(" ").append(toIndentedString(super.toString())).append("\n");{{/parent}} + {{#vars}}sb.append(" {{name}}: ").append(toIndentedString({{name}})).append("\n"); + {{/vars}}sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} \ No newline at end of file diff --git a/metadata-integration/java/spark-lineage/scripts/check_jar.sh b/metadata-integration/java/spark-lineage/scripts/check_jar.sh index 73f4bdd0f3f43..58b4155c26066 100755 --- a/metadata-integration/java/spark-lineage/scripts/check_jar.sh +++ b/metadata-integration/java/spark-lineage/scripts/check_jar.sh @@ -12,6 +12,17 @@ jar -tvf $jarFile |\ grep -v "legacyPegasusSchemas/" |\ grep -v " com/$" |\ grep -v "git.properties" |\ + grep -v " org/$" |\ + grep -v " io/$" |\ + grep -v "git.properties" |\ + grep -v "org/springframework" |\ + grep -v "org/aopalliance" |\ + grep -v "javax/" |\ + grep -v "io/swagger" |\ + grep -v "JavaSpring" |\ + grep -v "java-header-style.xml" |\ + grep -v "xml-header-style.xml" |\ + grep -v "license.header" |\ grep -v "client.properties" if [ $? -ne 0 ]; then diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index fc8831e7ec957..71acc943868cb 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -1,4 +1,5 @@ apply plugin: 'java' +apply plugin: 'org.hidetake.swagger.generator' configurations { enhance @@ -25,6 +26,7 @@ dependencies { compile externalDependency.elasticSearchRest compile externalDependency.elasticSearchTransport compile externalDependency.javatuples + compile externalDependency.javaxValidation compile externalDependency.kafkaClients compile externalDependency.ebean enhance externalDependency.ebeanAgent @@ -32,6 +34,7 @@ dependencies { compile externalDependency.resilience4j compile externalDependency.springContext compile externalDependency.swaggerAnnotations + swaggerCodegen 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.33' annotationProcessor externalDependency.lombok @@ -81,3 +84,29 @@ project.compileJava { transformArgs: 'debug=1') } } + +tasks.register('generateOpenApiPojos', GenerateSwaggerCode) { + it.setInputFile( + file( + "${project(':metadata-models').projectDir}/src/generatedJsonSchema/combined/open-api.yaml" + ) + ) + it.setOutputDir(file("$projectDir/generated")) + it.setLanguage("spring") + it.setComponents(['models']) + it.setTemplateDir(file("$projectDir/src/main/resources/JavaSpring")) + it.setAdditionalProperties([ + "group-id" : "io.datahubproject", + "dateLibrary" : "java8", + "java8" : "true", + "modelPropertyNaming" : "original", + "modelPackage" : "io.datahubproject.openapi.generated"] as Map) + + dependsOn ':metadata-models:generateJsonSchema' +} + +compileJava.dependsOn generateOpenApiPojos +sourceSets.main.java.srcDir "${generateOpenApiPojos.outputDir}/src/main/java" +sourceSets.main.resources.srcDir "${generateOpenApiPojos.outputDir}/src/main/resources" + +checkstyleMain.exclude '**/generated/**' diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index ded5c6dd14318..ac5275fa4bab4 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -16,6 +16,7 @@ import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.JacksonDataTemplateCodec; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.entity.AspectType; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; @@ -728,6 +729,9 @@ private Map getEnvelopedAspects(final final EnvelopedAspect envelopedAspect = new EnvelopedAspect(); envelopedAspect.setName(currAspectEntry.getKey().getAspect()); envelopedAspect.setVersion(currAspectEntry.getKey().getVersion()); + // TODO: I think we can assume this here, adding as it's a required field so object mapping barfs when trying to access it, + // since nowhere else is using it should be safe for now at least + envelopedAspect.setType(AspectType.VERSIONED); envelopedAspect.setValue(aspect); envelopedAspect.setCreated(new AuditStamp() .setActor(Urn.createFromString(currAspectEntry.getCreatedBy())) @@ -749,6 +753,9 @@ private EnvelopedAspect getKeyEnvelopedAspect(final Urn urn) throws URISyntaxExc envelopedAspect.setName(keySpec.getName()); envelopedAspect.setVersion(ASPECT_LATEST_VERSION); envelopedAspect.setValue(aspect); + // TODO: I think we can assume this here, adding as it's a required field so object mapping barfs when trying to access it, + // since nowhere else is using it should be safe for now at least + envelopedAspect.setType(AspectType.VERSIONED); envelopedAspect.setCreated( new AuditStamp().setActor(Urn.createFromString(SYSTEM_ACTOR)).setTime(System.currentTimeMillis())); diff --git a/metadata-io/src/main/resources/JavaSpring/interface.mustache b/metadata-io/src/main/resources/JavaSpring/interface.mustache new file mode 100644 index 0000000000000..ae2faa2ce0b49 --- /dev/null +++ b/metadata-io/src/main/resources/JavaSpring/interface.mustache @@ -0,0 +1,21 @@ +{{#jackson}} +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +{{/jackson}} +/** +* {{#description}}{{.}}{{/description}}{{^description}}{{classname}}{{/description}} +*/ +{{#jackson}} +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.EXISTING_PROPERTY, + property = "__type") +@JsonSubTypes({ + {{#subTypes}} + @JsonSubTypes.Type(value = {{classname}}.class, name = "{{classname}}"){{^@last}},{{/@last}} + {{/subTypes}} +}) +{{/jackson}} +public interface {{{classname}}} { + +} \ No newline at end of file diff --git a/metadata-io/src/main/resources/JavaSpring/model.mustache b/metadata-io/src/main/resources/JavaSpring/model.mustache new file mode 100644 index 0000000000000..a048f249a6b3d --- /dev/null +++ b/metadata-io/src/main/resources/JavaSpring/model.mustache @@ -0,0 +1,41 @@ +package {{package}}; + +{{^x-is-composed-model}} +import java.util.Objects; +{{#imports}}import {{import}}; +{{/imports}} +{{#serializableModel}} +import java.io.Serializable; +{{/serializableModel}} +{{#useBeanValidation}} +import org.springframework.validation.annotation.Validated; +import javax.validation.Valid; +import com.fasterxml.jackson.annotation.JsonInclude; +import javax.validation.constraints.*; +{{/useBeanValidation}} +{{#jackson}} +{{#withXml}} +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +{{/withXml}} +{{/jackson}} +{{#withXml}} +import javax.xml.bind.annotation.*; +{{/withXml}} +{{/x-is-composed-model}} + +{{#models}} +{{#model}} +{{#isComposedModel}} +{{>interface}} +{{/isComposedModel}} +{{^isComposedModel}} +{{#isEnum}} +{{>enumOuterClass}} +{{/isEnum}} +{{^isEnum}} +{{>pojo}} +{{/isEnum}} +{{/isComposedModel}} +{{/model}} +{{/models}} \ No newline at end of file diff --git a/metadata-io/src/main/resources/JavaSpring/pojo.mustache b/metadata-io/src/main/resources/JavaSpring/pojo.mustache new file mode 100644 index 0000000000000..8747765dcb031 --- /dev/null +++ b/metadata-io/src/main/resources/JavaSpring/pojo.mustache @@ -0,0 +1,166 @@ +{{#if hasVars}} +{{else}} +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +{{/if}} +/** + * {{#description}}{{.}}{{/description}}{{^description}}{{classname}}{{/description}} + */{{#description}} +{{#useOas2}}@ApiModel{{/useOas2}}{{^useOas2}}@Schema{{/useOas2}}(description = "{{{description}}}"){{/description}} +{{#useBeanValidation}}@Validated{{/useBeanValidation}} +{{>generatedAnnotation}}{{#discriminator}}{{>typeInfoAnnotation}}{{/discriminator}}{{>xmlAnnotation}} +@JsonInclude(JsonInclude.Include.NON_NULL) +public class {{classname}} {{#parent}}extends {{{parent}}}{{/parent}} {{#serializableModel}}implements Serializable {{#interfaceModels}}, {{classname}}{{^@last}}, {{/@last}}{{#@last}} {{/@last}}{{/interfaceModels}}{{/serializableModel}}{{^serializableModel}}{{#interfaceModels}}{{#@first}}implements {{/@first}}{{classname}}{{^@last}}, {{/@last}}{{#@last}}{{/@last}}{{/interfaceModels}}{{/serializableModel}} { +{{#serializableModel}} + private static final long serialVersionUID = 1L; + +{{/serializableModel}} +{{#if interfaceModels}} + + @JsonProperty(value = "__type", defaultValue = "{{classname}}") + private String __type = "{{classname}}"; + + /** + * Name of this subclass in SimpleClassName format + * @return __type + **/ + @Schema(required = true, description = "Name of this subclass in SimpleClassName format", allowableValues = {"{{classname}}"}, + defaultValue = "{{classname}}") + @NotNull + public String get__type() { + return __type; + } +{{/if}} + + {{#vars}} + {{#baseItems this}} + {{#isEnum}} +{{>enumClass}} + {{/isEnum}} + {{/baseItems}} + {{#jackson}} + {{#vendorExtensions.x-is-discriminator-property}} + @JsonTypeId + {{/vendorExtensions.x-is-discriminator-property}} + {{^vendorExtensions.x-is-discriminator-property}} + @JsonProperty("{{baseName}}"){{#withXml}} + @JacksonXmlProperty({{#isXmlAttribute}}isAttribute = true, {{/isXmlAttribute}}{{#xmlNamespace}}namespace="{{xmlNamespace}}", {{/xmlNamespace}}localName = "{{#xmlName}}{{xmlName}}{{/xmlName}}{{^xmlName}}{{baseName}}{{/xmlName}}"){{/withXml}} + {{/vendorExtensions.x-is-discriminator-property}} + {{/jackson}} + {{#gson}} + @SerializedName("{{baseName}}") + {{/gson}} + {{#isContainer}} + {{#useBeanValidation}}@Valid{{/useBeanValidation}} + private {{{datatypeWithEnum}}} {{name}}{{#required}} = {{{defaultValue}}}{{/required}}{{^required}} = null{{/required}}; + {{/isContainer}} + {{^isContainer}} + private {{{datatypeWithEnum}}} {{name}} = {{{defaultValue}}}; + {{/isContainer}} + + {{/vars}} + {{#vars}} + public {{classname}} {{name}}({{{datatypeWithEnum}}} {{name}}) { + this.{{name}} = {{name}}; + return this; + } + {{#isListContainer}} + + public {{classname}} add{{nameInCamelCase}}Item({{{items.datatypeWithEnum}}} {{name}}Item) { + {{^required}} + if (this.{{name}} == null) { + this.{{name}} = {{{defaultValue}}}; + } + {{/required}} + this.{{name}}.add({{name}}Item); + return this; + } + {{/isListContainer}} + {{#isMapContainer}} + + public {{classname}} put{{nameInCamelCase}}Item(String key, {{{items.datatypeWithEnum}}} {{name}}Item) { + {{^required}} + if (this.{{name}} == null) { + this.{{name}} = {{{defaultValue}}}; + } + {{/required}} + this.{{name}}.put(key, {{name}}Item); + return this; + } + {{/isMapContainer}} + + /** + {{#description}} + * {{{description}}} + {{/description}} + {{^description}} + * Get {{name}} + {{/description}} + {{#minimum}} + * minimum: {{minimum}} + {{/minimum}} + {{#maximum}} + * maximum: {{maximum}} + {{/maximum}} + * @return {{name}} + **/ + {{#vendorExtensions.extraAnnotation}} + {{{vendorExtensions.extraAnnotation}}} + {{/vendorExtensions.extraAnnotation}} + {{#useOas2}} + @ApiModelProperty({{#example}}example = "{{{example}}}", {{/example}}{{#required}}required = {{required}}, {{/required}}{{#isReadOnly}}readOnly = {{{isReadOnly}}}, {{/isReadOnly}}value = "{{{description}}}") + {{/useOas2}} + {{^useOas2}} + @Schema({{#example}}example = "{{{example}}}", {{/example}}{{#required}}required = {{required}}, {{/required}}{{#isReadOnly}}accessMode = Schema.AccessMode.READ_ONLY, {{/isReadOnly}}description = "{{{description}}}") + {{/useOas2}} + {{#useBeanValidation}}{{>beanValidation}}{{/useBeanValidation}} public {{{datatypeWithEnum}}} {{getter}}() { + return {{name}}; + } + + public void {{setter}}({{{datatypeWithEnum}}} {{name}}) { + this.{{name}} = {{name}}; + } + + {{/vars}} + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + }{{#hasVars}} + {{classname}} {{classVarName}} = ({{classname}}) o; + return {{#vars}}Objects.equals(this.{{name}}, {{classVarName}}.{{name}}){{#hasMore}} && + {{/hasMore}}{{/vars}}{{#parent}} && + super.equals(o){{/parent}};{{/hasVars}}{{^hasVars}} + return true;{{/hasVars}} + } + + @Override + public int hashCode() { + return Objects.hash({{#vars}}{{name}}{{#hasMore}}, {{/hasMore}}{{/vars}}{{#parent}}{{#hasVars}}, {{/hasVars}}super.hashCode(){{/parent}}); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class {{classname}} {\n"); + {{#parent}}sb.append(" ").append(toIndentedString(super.toString())).append("\n");{{/parent}} + {{#vars}}sb.append(" {{name}}: ").append(toIndentedString({{name}})).append("\n"); + {{/vars}}sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} \ No newline at end of file diff --git a/metadata-models/build.gradle b/metadata-models/build.gradle index a9e58b2f63c53..12e9711296f26 100644 --- a/metadata-models/build.gradle +++ b/metadata-models/build.gradle @@ -1,3 +1,6 @@ +import io.datahubproject.GenerateJsonSchemaTask + + apply plugin: 'pegasus' @@ -13,3 +16,9 @@ mainAvroSchemaJar.dependsOn generateAvroSchema pegasus.main.generationModes = [PegasusGenerationMode.PEGASUS, PegasusGenerationMode.AVRO] + +tasks.register('generateJsonSchema', GenerateJsonSchemaTask) { + it.setInputDirectory("$projectDir/src/mainGeneratedAvroSchema") + it.setOutputDirectory("$projectDir/src/generatedJsonSchema") + dependsOn generateAvroSchema +} \ No newline at end of file diff --git a/metadata-service/factories/src/main/resources/application.yml b/metadata-service/factories/src/main/resources/application.yml index e181a286efb12..05f689cd0e617 100644 --- a/metadata-service/factories/src/main/resources/application.yml +++ b/metadata-service/factories/src/main/resources/application.yml @@ -169,3 +169,7 @@ spring: mvc: servlet: path: /openapi + +springdoc: + cache: + disabled: true diff --git a/metadata-service/openapi-servlet/build.gradle b/metadata-service/openapi-servlet/build.gradle index 091fa567c73cb..f5cdfe42cbff6 100644 --- a/metadata-service/openapi-servlet/build.gradle +++ b/metadata-service/openapi-servlet/build.gradle @@ -4,6 +4,7 @@ dependencies { compile project(':metadata-service:factories') + compile externalDependency.reflections compile externalDependency.springBoot compile externalDependency.springCore compile externalDependency.springDocUI @@ -12,6 +13,10 @@ dependencies { compile externalDependency.springBeans compile externalDependency.springContext compile externalDependency.lombok + compile externalDependency.antlr4 annotationProcessor externalDependency.lombok + + testCompile externalDependency.testng + testCompile externalDependency.mockito } \ No newline at end of file diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/GlobalControllerExceptionHandler.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/GlobalControllerExceptionHandler.java similarity index 92% rename from metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/GlobalControllerExceptionHandler.java rename to metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/GlobalControllerExceptionHandler.java index 73667895ce309..47e2cfec3a9c0 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/GlobalControllerExceptionHandler.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/GlobalControllerExceptionHandler.java @@ -1,4 +1,4 @@ -package io.datahubproject.openapi.timeline; +package io.datahubproject.openapi; import org.springframework.core.convert.ConversionFailedException; import org.springframework.http.HttpStatus; diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/config/SpringWebConfig.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java similarity index 91% rename from metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/config/SpringWebConfig.java rename to metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java index 7cb618836731b..6148149ca6da4 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/config/SpringWebConfig.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java @@ -1,6 +1,6 @@ -package io.datahubproject.openapi.timeline.config; +package io.datahubproject.openapi.config; -import io.datahubproject.openapi.timeline.converter.StringToChangeCategoryConverter; +import io.datahubproject.openapi.converter.StringToChangeCategoryConverter; import io.swagger.v3.oas.annotations.OpenAPIDefinition; import io.swagger.v3.oas.annotations.servers.Server; import java.util.List; diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/converter/StringToChangeCategoryConverter.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/converter/StringToChangeCategoryConverter.java similarity index 96% rename from metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/converter/StringToChangeCategoryConverter.java rename to metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/converter/StringToChangeCategoryConverter.java index ae363aa8d1f03..e88f499208af8 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/converter/StringToChangeCategoryConverter.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/converter/StringToChangeCategoryConverter.java @@ -1,4 +1,4 @@ -package io.datahubproject.openapi.timeline.converter; +package io.datahubproject.openapi.converter; import com.linkedin.metadata.timeline.data.ChangeCategory; import java.util.List; diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/dto/RollbackRunResultDto.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/dto/RollbackRunResultDto.java new file mode 100644 index 0000000000000..0be69e3264957 --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/dto/RollbackRunResultDto.java @@ -0,0 +1,16 @@ +package io.datahubproject.openapi.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import io.datahubproject.openapi.generated.AspectRowSummary; +import java.util.List; +import lombok.Builder; +import lombok.Value; + + +@Value +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +public class RollbackRunResultDto { + List rowsRolledBack; + Integer rowsDeletedFromEntityDeletion; +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/dto/UpsertAspectRequest.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/dto/UpsertAspectRequest.java new file mode 100644 index 0000000000000..67858581ba97a --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/dto/UpsertAspectRequest.java @@ -0,0 +1,39 @@ +package io.datahubproject.openapi.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import io.datahubproject.openapi.generated.OneOfGenericAspectValue; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Builder; +import lombok.Value; + + +@JsonInclude(JsonInclude.Include.NON_NULL) +@Value +@Builder +@JsonDeserialize(builder = UpsertAspectRequest.UpsertAspectRequestBuilder.class) +public class UpsertAspectRequest { + + @JsonProperty("entityType") + @Schema(required = true, description = "The name of the entity matching with its definition in the entity registry") + String entityType; + + @JsonProperty("entityUrn") + @Schema(description = "Urn of the entity to be updated with the corresponding aspect, required if entityKey is null") + String entityUrn; + + @JsonProperty("entityKeyAspect") + @Schema(description = "A key aspect referencing the entity to be updated, required if entityUrn is null") + OneOfGenericAspectValue entityKeyAspect; + + @JsonProperty("aspect") + @Schema(required = true, description = "Aspect value to be upserted") + OneOfGenericAspectValue aspect; + + @JsonPOJOBuilder(withPrefix = "") + public static class UpsertAspectRequestBuilder { + + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/dto/UrnResponseMap.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/dto/UrnResponseMap.java new file mode 100644 index 0000000000000..02be0cc93eb1c --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/dto/UrnResponseMap.java @@ -0,0 +1,17 @@ +package io.datahubproject.openapi.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.datahubproject.openapi.generated.EntityResponse; +import java.util.Map; +import lombok.Builder; +import lombok.Value; + + +@Value +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +public class UrnResponseMap { + @JsonProperty("responses") + private Map responses; +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/entities/EntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/entities/EntitiesController.java new file mode 100644 index 0000000000000..fd73553e0e58c --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/entities/EntitiesController.java @@ -0,0 +1,167 @@ +package io.datahubproject.openapi.entities; + +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.utils.metrics.MetricUtils; +import com.linkedin.util.Pair; +import io.datahubproject.openapi.dto.RollbackRunResultDto; +import io.datahubproject.openapi.dto.UpsertAspectRequest; +import io.datahubproject.openapi.dto.UrnResponseMap; +import io.datahubproject.openapi.generated.AspectRowSummary; +import io.datahubproject.openapi.util.MappingUtil; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.net.URLDecoder; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.propertyeditors.StringArrayPropertyEditor; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.WebDataBinder; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.InitBinder; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import static com.linkedin.metadata.utils.PegasusUtils.*; + + +@RestController +@AllArgsConstructor +@RequestMapping("/entities/v1") +@Slf4j +@Tag(name = "Entities", description = "APIs for ingesting and accessing entities and their constituent aspects") +public class EntitiesController { + + private final EntityService _entityService; + private final ObjectMapper _objectMapper; + + @InitBinder + public void initBinder(WebDataBinder binder) { + binder.registerCustomEditor(String[].class, new StringArrayPropertyEditor(null)); + } + + @GetMapping(value = "/latest", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity getEntities( + @Parameter(name = "urns", required = true, description = "A list of raw urn strings, only supports a single entity type per request.") + @RequestParam("urns") @Nonnull String[] urns, + @Parameter(name = "aspectNames", description = "The list of aspect names to retrieve") + @RequestParam(name = "aspectNames", required = false) @Nullable String[] aspectNames) { + Timer.Context context = MetricUtils.timer("getEntities").time(); + final Set entityUrns = + Arrays.stream(urns) + // Have to decode here because of frontend routing, does No-op for already unencoded through direct API access + .map(URLDecoder::decode) + .map(UrnUtils::getUrn).collect(Collectors.toSet()); + log.debug("GET ENTITIES {}", entityUrns); + if (entityUrns.size() <= 0) { + return ResponseEntity.ok(UrnResponseMap.builder().responses(Collections.emptyMap()).build()); + } + // TODO: Only supports one entity type at a time, may cause confusion + final String entityName = urnToEntityName(entityUrns.iterator().next()); + final Set projectedAspects = aspectNames == null ? _entityService.getEntityAspectNames(entityName) + : new HashSet<>(Arrays.asList(aspectNames)); + Throwable exceptionally = null; + try { + return ResponseEntity.ok(UrnResponseMap.builder() + .responses(MappingUtil.mapServiceResponse(_entityService + .getEntitiesV2(entityName, entityUrns, projectedAspects), _objectMapper)) + .build()); + } catch (Exception e) { + exceptionally = e; + throw new RuntimeException( + String.format("Failed to batch get entities with urns: %s, projectedAspects: %s", entityUrns, + projectedAspects), e); + } finally { + if (exceptionally != null) { + MetricUtils.counter(MetricRegistry.name("getEntities", "failed")).inc(); + } else { + MetricUtils.counter(MetricRegistry.name("getEntities", "success")).inc(); + } + context.stop(); + } + } + + @PostMapping(value = "/", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity> postEntities( + @RequestBody @Nonnull List aspectRequests) { + log.info("INGEST PROPOSAL proposal: {}", aspectRequests); + + List> responses = aspectRequests.stream() + .map(MappingUtil::mapToProposal) + .map(proposal -> MappingUtil.ingestProposal(proposal, _entityService, _objectMapper)) + .collect(Collectors.toList()); + if (responses.stream().anyMatch(Pair::getSecond)) { + return ResponseEntity.status(HttpStatus.CREATED) + .body(responses.stream().filter(Pair::getSecond).map(Pair::getFirst).collect(Collectors.toList())); + } else { + return ResponseEntity.ok(Collections.emptyList()); + } + } + + @DeleteMapping(value = "/", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity> deleteEntities( + @Parameter(name = "urns", required = true, description = "A list of raw urn strings, only supports a single entity type per request.") + @RequestParam("urns") @Nonnull String[] urns, + @Parameter(name = "soft", description = "Determines whether the delete will be soft or hard, defaults to true for soft delete") + @RequestParam(value = "soft", defaultValue = "true") boolean soft) { + Timer.Context context = MetricUtils.timer("deleteEntities").time(); + final Set entityUrns = + Arrays.stream(urns) + // Have to decode here because of frontend routing, does No-op for already unencoded through direct API access + .map(URLDecoder::decode) + .map(UrnUtils::getUrn).collect(Collectors.toSet()); + Throwable exceptionally = null; + try { + if (!soft) { + + return ResponseEntity.ok(entityUrns.stream() + .map(_entityService::deleteUrn) + .map(rollbackRunResult -> MappingUtil.mapRollbackRunResult(rollbackRunResult, _objectMapper)) + .collect(Collectors.toList())); + } else { + List deleteRequests = entityUrns.stream() + .map(entityUrn -> MappingUtil.createStatusRemoval(entityUrn, _entityService)) + .collect(Collectors.toList()); + return ResponseEntity.ok(Collections.singletonList(RollbackRunResultDto.builder() + .rowsRolledBack(deleteRequests.stream() + .map(MappingUtil::mapToProposal) + .map(proposal -> MappingUtil.ingestProposal(proposal, _entityService, _objectMapper)) + .filter(Pair::getSecond) + .map(Pair::getFirst) + .map(urnString -> new AspectRowSummary().urn(urnString)) + .collect(Collectors.toList())) + .rowsDeletedFromEntityDeletion(deleteRequests.size()) + .build())); + } + } catch (Exception e) { + exceptionally = e; + throw new RuntimeException( + String.format("Failed to batch delete entities with urns: %s", entityUrns), e); + } finally { + if (exceptionally != null) { + MetricUtils.counter(MetricRegistry.name("getEntities", "failed")).inc(); + } else { + MetricUtils.counter(MetricRegistry.name("getEntities", "success")).inc(); + } + context.stop(); + } + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/platform/entities/PlatformEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/platform/entities/PlatformEntitiesController.java new file mode 100644 index 0000000000000..0d7ed888c8f8b --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/platform/entities/PlatformEntitiesController.java @@ -0,0 +1,57 @@ +package io.datahubproject.openapi.platform.entities; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.util.Pair; +import io.datahubproject.openapi.generated.MetadataChangeProposal; +import io.datahubproject.openapi.util.MappingUtil; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.propertyeditors.StringArrayPropertyEditor; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.WebDataBinder; +import org.springframework.web.bind.annotation.InitBinder; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + + +@RestController +@AllArgsConstructor +@RequestMapping("/platform/entities/v1") +@Slf4j +@Tag(name = "Platform Entities", description = "Platform level APIs intended for lower level access to entities") +public class PlatformEntitiesController { + + private final EntityService _entityService; + private final ObjectMapper _objectMapper; + + @InitBinder + public void initBinder(WebDataBinder binder) { + binder.registerCustomEditor(String[].class, new StringArrayPropertyEditor(null)); + } + + @PostMapping(value = "/", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity> postEntities( + @RequestBody @Nonnull List metadataChangeProposals) { + log.info("INGEST PROPOSAL proposal: {}", metadataChangeProposals); + + List> responses = metadataChangeProposals.stream() + .map(proposal -> MappingUtil.ingestProposal(proposal, _entityService, _objectMapper)) + .collect(Collectors.toList()); + if (responses.stream().anyMatch(Pair::getSecond)) { + return ResponseEntity.status(HttpStatus.CREATED) + .body(responses.stream().filter(Pair::getSecond).map(Pair::getFirst).collect(Collectors.toList())); + } else { + return ResponseEntity.ok(Collections.emptyList()); + } + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/TimelineController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/TimelineController.java index 55d6cbf734058..dd8a58b34f324 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/TimelineController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/timeline/TimelineController.java @@ -5,6 +5,7 @@ import com.linkedin.metadata.timeline.TimelineService; import com.linkedin.metadata.timeline.data.ChangeCategory; import com.linkedin.metadata.timeline.data.ChangeTransaction; +import io.swagger.v3.oas.annotations.tags.Tag; import java.net.URISyntaxException; import java.util.List; import java.util.Set; @@ -21,6 +22,7 @@ @RestController @AllArgsConstructor @RequestMapping("/timeline/v1") +@Tag(name = "Timeline", description = "An API for retrieving historical updates to entities and their related documentation.") public class TimelineController { private final TimelineService _timelineService; @@ -28,13 +30,13 @@ public class TimelineController { /** * * @param rawUrn - * @param start - * @param end - * @param startVersionStamp - * @param endVersionStamp + * @param startTime + * @param endTime * @param raw * @param categories * @return + * @throws URISyntaxException + * @throws JsonProcessingException */ @GetMapping(path = "/{urn}", produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity> getTimeline( diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java new file mode 100644 index 0000000000000..8d07196535322 --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java @@ -0,0 +1,372 @@ +package io.datahubproject.openapi.util; + +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.linkedin.avro2pegasus.events.KafkaAuditHeader; +import com.linkedin.avro2pegasus.events.UUID; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.ByteString; +import com.linkedin.data.DataMap; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.entity.Aspect; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.RollbackRunResult; +import com.linkedin.metadata.entity.ValidationException; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.resources.entity.AspectUtils; +import com.linkedin.metadata.utils.metrics.MetricUtils; +import com.linkedin.mxe.GenericAspect; +import com.linkedin.mxe.SystemMetadata; +import com.linkedin.util.Pair; +import io.datahubproject.openapi.dto.RollbackRunResultDto; +import io.datahubproject.openapi.dto.UpsertAspectRequest; +import io.datahubproject.openapi.generated.AspectRowSummary; +import io.datahubproject.openapi.generated.AspectType; +import io.datahubproject.openapi.generated.AuditStamp; +import io.datahubproject.openapi.generated.EntityResponse; +import io.datahubproject.openapi.generated.EnvelopedAspect; +import io.datahubproject.openapi.generated.MetadataChangeProposal; +import io.datahubproject.openapi.generated.OneOfEnvelopedAspectValue; +import io.datahubproject.openapi.generated.OneOfGenericAspectValue; +import io.datahubproject.openapi.generated.Status; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; +import org.reflections.Reflections; +import org.reflections.scanners.ResourcesScanner; +import org.reflections.scanners.SubTypesScanner; +import org.reflections.util.ClasspathHelper; +import org.reflections.util.ConfigurationBuilder; +import org.reflections.util.FilterBuilder; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider; +import org.springframework.core.type.filter.AssignableTypeFilter; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.web.client.HttpClientErrorException; + +import static com.linkedin.metadata.Constants.*; +import static java.nio.charset.StandardCharsets.*; + +@Slf4j +public class MappingUtil { + private MappingUtil() { + + } + + private static final Map> ENVELOPED_ASPECT_TYPE_MAP = + new HashMap<>(); + private static final Map, String> ASPECT_NAME_MAP = + new HashMap<>(); + private static final Map> PEGASUS_TYPE_MAP = new HashMap<>(); + private static final Pattern CLASS_NAME_PATTERN = + Pattern.compile("(\"com\\.linkedin\\.)([a-z]+?\\.)+?(?[A-Z]\\w+?)(\":\\{)(?.*?)(}})"); + private static final Pattern GLOBAL_TAGS_PATTERN = + Pattern.compile("\"globalTags\":\\{"); + private static final Pattern GLOSSARY_TERMS_PATTERN = + Pattern.compile("\"glossaryTerms\":\\{"); + + private static final String DISCRIMINATOR = "__type"; + private static final Pattern CLASS_TYPE_NAME_PATTERN = + Pattern.compile("(\\s+?\"__type\"\\s+?:\\s+?\")(?\\w*?)(\"[,]?\\s+?)(?[\\S\\s]*?)(\\s+})"); + private static final String PEGASUS_PACKAGE = "com.linkedin"; + private static final String GLOBAL_TAGS = "GlobalTags"; + private static final String GLOSSARY_TERMS = "GlossaryTerms"; + + static { + // Build a map from __type name to generated class + ClassPathScanningCandidateComponentProvider provider = new ClassPathScanningCandidateComponentProvider(false); + provider.addIncludeFilter(new AssignableTypeFilter(OneOfEnvelopedAspectValue.class)); + Set components = provider.findCandidateComponents("io/datahubproject/openapi/generated"); + components.forEach(MappingUtil::putEnvelopedAspectEntry); + + provider = new ClassPathScanningCandidateComponentProvider(false); + provider.addIncludeFilter(new AssignableTypeFilter(OneOfGenericAspectValue.class)); + components = provider.findCandidateComponents("io/datahubproject/openapi/generated"); + components.forEach(MappingUtil::putGenericAspectEntry); + + List classLoadersList = new ArrayList<>(); + classLoadersList.add(ClasspathHelper.contextClassLoader()); + classLoadersList.add(ClasspathHelper.staticClassLoader()); + + // Build a map from fully qualified Pegasus generated class name to class + Reflections reflections = new Reflections(new ConfigurationBuilder() + .setScanners(new SubTypesScanner(false), new ResourcesScanner()) + .setUrls(ClasspathHelper.forClassLoader(classLoadersList.toArray(new ClassLoader[0]))) + .filterInputsBy(new FilterBuilder().include(FilterBuilder.prefix(PEGASUS_PACKAGE)))); + Set> pegasusComponents = reflections.getSubTypesOf(RecordTemplate.class); + pegasusComponents.forEach(aClass -> PEGASUS_TYPE_MAP.put(aClass.getSimpleName(), aClass)); + } + + public static Map mapServiceResponse(Map serviceResponse, + ObjectMapper objectMapper) { + return serviceResponse.entrySet() + .stream() + .collect(Collectors.toMap(entry -> entry.getKey().toString(), entry -> mapEntityResponse(entry.getValue(), objectMapper))); + } + + public static EntityResponse mapEntityResponse(com.linkedin.entity.EntityResponse entityResponse, ObjectMapper objectMapper) { + return new EntityResponse().entityName(entityResponse.getEntityName()) + .urn(entityResponse.getUrn().toString()) + .aspects(entityResponse.getAspects() + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> mapEnvelopedAspect(entry.getValue(), objectMapper)))); + } + + public static EnvelopedAspect mapEnvelopedAspect(com.linkedin.entity.EnvelopedAspect envelopedAspect, + ObjectMapper objectMapper) { + return new EnvelopedAspect() + .name(envelopedAspect.getName()) + .timestamp(envelopedAspect.getTimestamp()) + .version(envelopedAspect.getVersion()) + .type(AspectType.fromValue(envelopedAspect.getType().name().toUpperCase(Locale.ROOT))) + .created(objectMapper.convertValue(envelopedAspect.getCreated().data(), AuditStamp.class)) + .value(mapAspectValue(envelopedAspect.getName(), envelopedAspect.getValue(), objectMapper)); + } + + public static OneOfEnvelopedAspectValue mapAspectValue(String aspectName, Aspect aspect, ObjectMapper objectMapper) { + Class aspectClass = ENVELOPED_ASPECT_TYPE_MAP.get(aspectName); + DataMap wrapper = aspect.data(); + wrapper.put(DISCRIMINATOR, aspectClass.getSimpleName()); + String dataMapAsJson; + try { + dataMapAsJson = objectMapper.writeValueAsString(wrapper); + Matcher classNameMatcher = CLASS_NAME_PATTERN.matcher(dataMapAsJson); + while (classNameMatcher.find()) { + String className = classNameMatcher.group("className"); + String content = classNameMatcher.group("content"); + StringBuilder replacement = new StringBuilder("\"" + DISCRIMINATOR + "\" : \"" + className + "\""); + + if (content.length() > 0) { + replacement.append(",") + .append(content); + } + replacement.append("}"); + dataMapAsJson = classNameMatcher.replaceFirst(Matcher.quoteReplacement(replacement.toString())); + classNameMatcher = CLASS_NAME_PATTERN.matcher(dataMapAsJson); + } + // Global Tags & Glossary Terms will not have the explicit class name in the DataMap, so we handle them differently + Matcher globalTagsMatcher = GLOBAL_TAGS_PATTERN.matcher(dataMapAsJson); + while (globalTagsMatcher.find()) { + String replacement = "\"globalTags\" : {\"" + DISCRIMINATOR + "\" : \"GlobalTags\","; + dataMapAsJson = globalTagsMatcher.replaceFirst(Matcher.quoteReplacement(replacement)); + globalTagsMatcher = GLOBAL_TAGS_PATTERN.matcher(dataMapAsJson); + } + Matcher glossaryTermsMatcher = GLOSSARY_TERMS_PATTERN.matcher(dataMapAsJson); + while (glossaryTermsMatcher.find()) { + String replacement = "\"glossaryTerms\" : {\"" + DISCRIMINATOR + "\" : \"GlossaryTerms\","; + dataMapAsJson = glossaryTermsMatcher.replaceFirst(Matcher.quoteReplacement(replacement)); + glossaryTermsMatcher = GLOSSARY_TERMS_PATTERN.matcher(dataMapAsJson); + } + return objectMapper.readValue(dataMapAsJson, aspectClass); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("unchecked") + private static void putEnvelopedAspectEntry(BeanDefinition beanDefinition) { + try { + Class cls = + (Class) Class.forName(beanDefinition.getBeanClassName()); + String aspectName = getAspectName(cls); + ENVELOPED_ASPECT_TYPE_MAP.put(aspectName, cls); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("unchecked") + private static void putGenericAspectEntry(BeanDefinition beanDefinition) { + try { + Class cls = + (Class) Class.forName(beanDefinition.getBeanClassName()); + String aspectName = getAspectName(cls); + ASPECT_NAME_MAP.put(cls, aspectName); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + private static String getAspectName(Class cls) { + char[] c = cls.getSimpleName().toCharArray(); + c[0] = Character.toLowerCase(c[0]); + return new String(c); + } + + + @Nonnull + public static GenericAspect convertGenericAspect(@Nonnull io.datahubproject.openapi.generated.GenericAspect genericAspect, + ObjectMapper objectMapper) { + try { + ObjectNode jsonTree = (ObjectNode) objectMapper.valueToTree(genericAspect).get("value"); + jsonTree.remove(DISCRIMINATOR); + String pretty = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonTree); + Matcher classTypeNameMatcher = CLASS_TYPE_NAME_PATTERN.matcher(pretty); + while (classTypeNameMatcher.find()) { + String classTypeName = classTypeNameMatcher.group("classTypeName"); + String content = classTypeNameMatcher.group("content"); + StringBuilder replacement = new StringBuilder(); + // Global Tags & Glossary Terms get used as both a union type and a non-union type, in the DataMap this means + // that it does not want the explicit class name if it is being used explicitly as a non-union type field on an aspect + if (!GLOBAL_TAGS.equals(classTypeName) && !GLOSSARY_TERMS.equals(classTypeName)) { + String pegasusClassName = PEGASUS_TYPE_MAP.get(classTypeName).getName(); + replacement.append("\"").append(pegasusClassName).append("\" : {"); + + if (content.length() > 0) { + replacement.append(content); + } + replacement.append("}}"); + } else { + replacement.append(content) + .append("}"); + } + pretty = classTypeNameMatcher.replaceFirst(Matcher.quoteReplacement(replacement.toString())); + classTypeNameMatcher = CLASS_TYPE_NAME_PATTERN.matcher(pretty); + } + return new GenericAspect().setContentType(genericAspect.getContentType()) + .setValue(ByteString.copyString(pretty, UTF_8)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + public static Pair ingestProposal(MetadataChangeProposal metadataChangeProposal, EntityService entityService, + ObjectMapper objectMapper) { + // TODO: Use the actor present in the IC. + Timer.Context context = MetricUtils.timer("postEntity").time(); + final com.linkedin.common.AuditStamp auditStamp = + new com.linkedin.common.AuditStamp().setTime(System.currentTimeMillis()) + .setActor(UrnUtils.getUrn(Constants.UNKNOWN_ACTOR)); + io.datahubproject.openapi.generated.KafkaAuditHeader auditHeader = metadataChangeProposal.getAuditHeader(); + + com.linkedin.mxe.MetadataChangeProposal serviceProposal = + new com.linkedin.mxe.MetadataChangeProposal() + .setEntityType(metadataChangeProposal.getEntityType()) + .setChangeType(ChangeType.valueOf(metadataChangeProposal.getChangeType().name())); + if (metadataChangeProposal.getEntityUrn() != null) { + serviceProposal.setEntityUrn(UrnUtils.getUrn(metadataChangeProposal.getEntityUrn())); + } + if (metadataChangeProposal.getSystemMetadata() != null) { + serviceProposal.setSystemMetadata( + objectMapper.convertValue(metadataChangeProposal.getSystemMetadata(), SystemMetadata.class)); + } + if (metadataChangeProposal.getAspectName() != null) { + serviceProposal.setAspectName(metadataChangeProposal.getAspectName()); + } + + if (auditHeader != null) { + KafkaAuditHeader kafkaAuditHeader = new KafkaAuditHeader(); + kafkaAuditHeader.setAuditVersion(auditHeader.getAuditVersion()) + .setTime(auditHeader.getTime()) + .setAppName(auditHeader.getAppName()) + .setMessageId(new UUID(ByteString.copyString(auditHeader.getMessageId(), UTF_8))) + .setServer(auditHeader.getServer()); + if (auditHeader.getInstance() != null) { + kafkaAuditHeader.setInstance(auditHeader.getInstance()); + } + if (auditHeader.getAuditVersion() != null) { + kafkaAuditHeader.setAuditVersion(auditHeader.getAuditVersion()); + } + if (auditHeader.getFabricUrn() != null) { + kafkaAuditHeader.setFabricUrn(auditHeader.getFabricUrn()); + } + if (auditHeader.getClusterConnectionString() != null) { + kafkaAuditHeader.setClusterConnectionString(auditHeader.getClusterConnectionString()); + } + serviceProposal.setAuditHeader(kafkaAuditHeader); + } + + serviceProposal = metadataChangeProposal.getEntityKeyAspect() != null + ? serviceProposal.setEntityKeyAspect( + MappingUtil.convertGenericAspect(metadataChangeProposal.getEntityKeyAspect(), objectMapper)) + : serviceProposal; + serviceProposal = metadataChangeProposal.getAspect() != null + ? serviceProposal.setAspect( + MappingUtil.convertGenericAspect(metadataChangeProposal.getAspect(), objectMapper)) + : serviceProposal; + + final List additionalChanges = + AspectUtils.getAdditionalChanges(serviceProposal, entityService); + + log.info("Proposal: {}", serviceProposal); + Throwable exceptionally = null; + try { + EntityService.IngestProposalResult proposalResult = entityService.ingestProposal(serviceProposal, auditStamp); + Urn urn = proposalResult.getUrn(); + additionalChanges.forEach(proposal -> entityService.ingestProposal(proposal, auditStamp)); + return new Pair<>(urn.toString(), proposalResult.isDidUpdate()); + } catch (ValidationException ve) { + exceptionally = ve; + throw HttpClientErrorException.create(HttpStatus.UNPROCESSABLE_ENTITY, ve.getMessage(), null, null, null); + } catch (Exception e) { + exceptionally = e; + throw e; + } finally { + if (exceptionally != null) { + MetricUtils.counter(MetricRegistry.name("postEntity", "failed")).inc(); + } else { + MetricUtils.counter(MetricRegistry.name("postEntity", "success")).inc(); + } + context.stop(); + } + } + + public static MetadataChangeProposal mapToProposal(UpsertAspectRequest aspectRequest) { + MetadataChangeProposal metadataChangeProposal = new MetadataChangeProposal(); + io.datahubproject.openapi.generated.GenericAspect + genericAspect = new io.datahubproject.openapi.generated.GenericAspect() + .value(aspectRequest.getAspect()) + .contentType(MediaType.APPLICATION_JSON_VALUE); + io.datahubproject.openapi.generated.GenericAspect keyAspect = null; + if (aspectRequest.getEntityKeyAspect() != null) { + keyAspect = new io.datahubproject.openapi.generated.GenericAspect() + .contentType(MediaType.APPLICATION_JSON_VALUE) + .value(aspectRequest.getEntityKeyAspect()); + } + metadataChangeProposal.aspect(genericAspect) + .changeType(io.datahubproject.openapi.generated.ChangeType.UPSERT) + .aspectName(ASPECT_NAME_MAP.get(aspectRequest.getAspect().getClass())) + .entityKeyAspect(keyAspect) + .entityUrn(aspectRequest.getEntityUrn()) + .entityType(aspectRequest.getEntityType()); + + return metadataChangeProposal; + } + + public static RollbackRunResultDto mapRollbackRunResult(RollbackRunResult rollbackRunResult, ObjectMapper objectMapper) { + List aspectRowSummaries = rollbackRunResult.getRowsRolledBack().stream() + .map(aspectRowSummary -> objectMapper.convertValue(aspectRowSummary.data(), AspectRowSummary.class)) + .collect(Collectors.toList()); + return RollbackRunResultDto.builder() + .rowsRolledBack(aspectRowSummaries) + .rowsDeletedFromEntityDeletion(rollbackRunResult.getRowsDeletedFromEntityDeletion()).build(); + } + + public static UpsertAspectRequest createStatusRemoval(Urn urn, EntityService entityService) { + EntitySpec entitySpec = entityService.getEntityRegistry().getEntitySpec(urn.getEntityType()); + if (entitySpec == null || !entitySpec.getAspectSpecMap().containsKey(STATUS_ASPECT_NAME)) { + throw new IllegalArgumentException("Entity type is not valid for soft deletes: " + urn.getEntityType()); + } + return UpsertAspectRequest.builder() + .aspect(new Status().removed(true)) + .entityUrn(urn.toString()) + .entityType(urn.getEntityType()) + .build(); + } +} diff --git a/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java b/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java new file mode 100644 index 0000000000000..b8ffe505f49c8 --- /dev/null +++ b/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java @@ -0,0 +1,154 @@ +package entities; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.metadata.event.EventProducer; +import com.linkedin.metadata.models.registry.EntityRegistry; +import io.datahubproject.openapi.dto.UpsertAspectRequest; +import io.datahubproject.openapi.entities.EntitiesController; +import io.datahubproject.openapi.generated.AuditStamp; +import io.datahubproject.openapi.generated.DatasetFieldProfile; +import io.datahubproject.openapi.generated.DatasetKey; +import io.datahubproject.openapi.generated.DatasetProfile; +import io.datahubproject.openapi.generated.FabricType; +import io.datahubproject.openapi.generated.GlobalTags; +import io.datahubproject.openapi.generated.GlossaryTermAssociation; +import io.datahubproject.openapi.generated.GlossaryTerms; +import io.datahubproject.openapi.generated.Histogram; +import io.datahubproject.openapi.generated.MySqlDDL; +import io.datahubproject.openapi.generated.SchemaField; +import io.datahubproject.openapi.generated.SchemaFieldDataType; +import io.datahubproject.openapi.generated.SchemaMetadata; +import io.datahubproject.openapi.generated.StringType; +import io.datahubproject.openapi.generated.SubTypes; +import io.datahubproject.openapi.generated.TagAssociation; +import io.datahubproject.openapi.generated.ViewProperties; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import mock.MockEntityRegistry; +import mock.MockEntityService; +import org.mockito.Mockito; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static com.linkedin.metadata.Constants.*; + + +public class EntitiesControllerTest { + + public static final String S = "somerandomstring"; + public static final String DATASET_URN = "urn:li:dataset:(urn:li:dataPlatform:platform,name,PROD)"; + public static final String CORPUSER_URN = "urn:li:corpuser:datahub"; + public static final String GLOSSARY_TERM_URN = "urn:li:glossaryTerm:SavingAccount"; + public static final String DATA_PLATFORM_URN = "urn:li:dataPlatform:platform"; + public static final String TAG_URN = "urn:li:tag:sometag"; + + @BeforeMethod + public void setup() + throws NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException { + EntityRegistry mockEntityRegistry = new MockEntityRegistry(); + EventProducer mockEntityEventProducer = Mockito.mock(EventProducer.class); + MockEntityService mockEntityService = new MockEntityService(mockEntityEventProducer, mockEntityRegistry); + _entitiesController = new EntitiesController(mockEntityService, new ObjectMapper()); + } + + EntitiesController _entitiesController; + + @Test + public void testIngestDataset() { + List datasetAspects = new ArrayList<>(); + UpsertAspectRequest viewProperties = UpsertAspectRequest.builder() + .aspect(new ViewProperties() + .viewLogic(S) + .viewLanguage(S) + .materialized(true)) + .entityType(DATASET_ENTITY_NAME) + .entityUrn(DATASET_URN) + .build(); + datasetAspects.add(viewProperties); + + UpsertAspectRequest subTypes = UpsertAspectRequest.builder() + .aspect(new SubTypes() + .typeNames(Collections.singletonList(S))) + .entityType(DATASET_ENTITY_NAME) + .entityKeyAspect(new DatasetKey() + .name("name") + .platform(DATA_PLATFORM_URN) + .origin(FabricType.PROD)) + .build(); + datasetAspects.add(subTypes); + + UpsertAspectRequest datasetProfile = UpsertAspectRequest.builder() + .aspect(new DatasetProfile().timestampMillis(0L).addFieldProfilesItem( + new DatasetFieldProfile() + .fieldPath(S) + .histogram(new Histogram() + .boundaries(Collections.singletonList(S)))) + ) + .entityType(DATASET_ENTITY_NAME) + .entityKeyAspect(new DatasetKey() + .name("name") + .platform(DATA_PLATFORM_URN) + .origin(FabricType.PROD)) + .build(); + datasetAspects.add(datasetProfile); + + UpsertAspectRequest schemaMetadata = UpsertAspectRequest.builder() + .aspect(new SchemaMetadata() + .schemaName(S) + .dataset(DATASET_URN) + .platform(DATA_PLATFORM_URN) + .hash(S) + .version(0L) + .platformSchema(new MySqlDDL().tableSchema(S)) + .fields(Collections.singletonList(new SchemaField() + .fieldPath(S) + .nativeDataType(S) + .type(new SchemaFieldDataType().type(new StringType())) + .description(S) + .globalTags(new GlobalTags() + .tags(Collections.singletonList(new TagAssociation() + .tag(TAG_URN)))) + .glossaryTerms(new GlossaryTerms() + .terms(Collections.singletonList(new GlossaryTermAssociation() + .urn(GLOSSARY_TERM_URN))) + .auditStamp(new AuditStamp() + .time(0L) + .actor(CORPUSER_URN))) + ) + )) + .entityType(DATASET_ENTITY_NAME) + .entityKeyAspect(new DatasetKey() + .name("name") + .platform(DATA_PLATFORM_URN) + .origin(FabricType.PROD)) + .build(); + datasetAspects.add(schemaMetadata); + + UpsertAspectRequest glossaryTerms = UpsertAspectRequest.builder() + .aspect(new GlossaryTerms() + .terms(Collections.singletonList(new GlossaryTermAssociation() + .urn(GLOSSARY_TERM_URN))) + .auditStamp(new AuditStamp() + .time(0L) + .actor(CORPUSER_URN))) + .entityType(DATASET_ENTITY_NAME) + .entityKeyAspect(new DatasetKey() + .name("name") + .platform(DATA_PLATFORM_URN) + .origin(FabricType.PROD)) + .build(); + datasetAspects.add(glossaryTerms); + + _entitiesController.postEntities(datasetAspects); + } + +// @Test +// public void testGetDataset() { +// _entitiesController.getEntities(new String[] {DATASET_URN}, +// new String[] { +// SCHEMA_METADATA_ASPECT_NAME +// }); +// } +} diff --git a/metadata-service/openapi-servlet/src/test/java/mock/MockAspectSpec.java b/metadata-service/openapi-servlet/src/test/java/mock/MockAspectSpec.java new file mode 100644 index 0000000000000..594bc583eeef0 --- /dev/null +++ b/metadata-service/openapi-servlet/src/test/java/mock/MockAspectSpec.java @@ -0,0 +1,27 @@ +package mock; + +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.RelationshipFieldSpec; +import com.linkedin.metadata.models.SearchScoreFieldSpec; +import com.linkedin.metadata.models.SearchableFieldSpec; +import com.linkedin.metadata.models.TimeseriesFieldCollectionSpec; +import com.linkedin.metadata.models.TimeseriesFieldSpec; +import com.linkedin.metadata.models.annotation.AspectAnnotation; +import java.util.List; +import javax.annotation.Nonnull; + + +public class MockAspectSpec extends AspectSpec { + public MockAspectSpec(@Nonnull AspectAnnotation aspectAnnotation, + @Nonnull List searchableFieldSpecs, + @Nonnull List searchScoreFieldSpecs, + @Nonnull List relationshipFieldSpecs, + @Nonnull List timeseriesFieldSpecs, + @Nonnull List timeseriesFieldCollectionSpecs, RecordDataSchema schema, + Class aspectClass) { + super(aspectAnnotation, searchableFieldSpecs, searchScoreFieldSpecs, relationshipFieldSpecs, timeseriesFieldSpecs, + timeseriesFieldCollectionSpecs, schema, aspectClass); + } +} diff --git a/metadata-service/openapi-servlet/src/test/java/mock/MockEntityRegistry.java b/metadata-service/openapi-servlet/src/test/java/mock/MockEntityRegistry.java new file mode 100644 index 0000000000000..28e4f4ab9042d --- /dev/null +++ b/metadata-service/openapi-servlet/src/test/java/mock/MockEntityRegistry.java @@ -0,0 +1,36 @@ +package mock; + +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.EventSpec; +import com.linkedin.metadata.models.registry.EntityRegistry; +import java.util.Collections; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + + +public class MockEntityRegistry implements EntityRegistry { + @Nonnull + @Override + public EntitySpec getEntitySpec(@Nonnull String entityName) { + return new MockEntitySpec(entityName); + } + + @Nullable + @Override + public EventSpec getEventSpec(@Nonnull String eventName) { + return null; + } + + @Nonnull + @Override + public Map getEntitySpecs() { + return Collections.emptyMap(); + } + + @Nonnull + @Override + public Map getEventSpecs() { + return null; + } +} diff --git a/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java b/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java new file mode 100644 index 0000000000000..60967adce1efd --- /dev/null +++ b/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java @@ -0,0 +1,198 @@ +package mock; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.GlobalTags; +import com.linkedin.common.GlossaryTermAssociation; +import com.linkedin.common.GlossaryTermAssociationArray; +import com.linkedin.common.GlossaryTerms; +import com.linkedin.common.TagAssociation; +import com.linkedin.common.TagAssociationArray; +import com.linkedin.common.UrnArray; +import com.linkedin.common.VersionedUrn; +import com.linkedin.common.urn.DatasetUrn; +import com.linkedin.common.urn.GlossaryTermUrn; +import com.linkedin.common.urn.TagUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.AspectType; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.metadata.aspect.VersionedAspect; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.ListResult; +import com.linkedin.metadata.entity.RollbackRunResult; +import com.linkedin.metadata.event.EventProducer; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.ListUrnsResult; +import com.linkedin.metadata.run.AspectRowSummary; +import com.linkedin.mxe.SystemMetadata; +import com.linkedin.schema.ForeignKeyConstraint; +import com.linkedin.schema.ForeignKeyConstraintArray; +import com.linkedin.schema.MySqlDDL; +import com.linkedin.schema.SchemaField; +import com.linkedin.schema.SchemaFieldArray; +import com.linkedin.schema.SchemaFieldDataType; +import com.linkedin.schema.SchemaMetadata; +import com.linkedin.schema.StringType; +import com.linkedin.util.Pair; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import javax.annotation.Nonnull; + +import static entities.EntitiesControllerTest.*; + + +public class MockEntityService extends EntityService { + public MockEntityService(@Nonnull EventProducer producer, @Nonnull EntityRegistry entityRegistry) { + super(producer, entityRegistry); + } + + @Override + public Map> getLatestAspects(@Nonnull Set urns, @Nonnull Set aspectNames) { + return null; + } + + @Override + public Map getLatestAspectsForUrn(@Nonnull Urn urn, @Nonnull Set aspectNames) { + return Collections.emptyMap(); + } + + @Override + public RecordTemplate getAspect(@Nonnull Urn urn, @Nonnull String aspectName, long version) { + return null; + } + + @Override + public Map> getLatestEnvelopedAspects(@Nonnull String entityName, @Nonnull Set urns, + @Nonnull Set aspectNames) throws URISyntaxException { + Urn urn = UrnUtils.getUrn(DATASET_URN); + Map> envelopedAspectMap = new HashMap<>(); + List aspects = new ArrayList<>(); + EnvelopedAspect schemaMetadata = new EnvelopedAspect(); + SchemaMetadata pegasusSchemaMetadata = new SchemaMetadata(); + pegasusSchemaMetadata.setDataset(DatasetUrn.createFromUrn(UrnUtils.getUrn(DATASET_URN))) + .setVersion(0L) + .setCreated(new AuditStamp().setActor(UrnUtils.getUrn(CORPUSER_URN)).setTime(System.currentTimeMillis())) + .setHash(S) + .setCluster(S) + .setPlatformSchema(SchemaMetadata.PlatformSchema.create(new MySqlDDL().setTableSchema(S))) + .setForeignKeys(new ForeignKeyConstraintArray(Collections.singletonList( + new ForeignKeyConstraint() + .setForeignDataset(urn) + .setName(S) + .setForeignFields(new UrnArray(Collections.singletonList(urn)))))) + .setFields(new SchemaFieldArray(Collections.singletonList( + new SchemaField() + .setDescription(S) + .setFieldPath(S) + .setType(new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType()))) + .setGlobalTags( + new GlobalTags() + .setTags(new TagAssociationArray(Collections.singletonList( + new TagAssociation().setTag(TagUrn.createFromUrn(UrnUtils.getUrn(TAG_URN))) + )))) + .setGlossaryTerms(new GlossaryTerms().setTerms( + new GlossaryTermAssociationArray(Collections.singletonList( + new GlossaryTermAssociation() + .setUrn(GlossaryTermUrn.createFromUrn(UrnUtils.getUrn(GLOSSARY_TERM_URN))) + ))) + ) + )) + ); + schemaMetadata + .setType(AspectType.VERSIONED) + .setName("schemaMetadata") + .setValue(new Aspect(pegasusSchemaMetadata.data())); + aspects.add(schemaMetadata); + envelopedAspectMap.put(UrnUtils.getUrn(DATASET_URN), aspects); + return envelopedAspectMap; + } + + @Override + public Map> getVersionedEnvelopedAspects(@Nonnull Set versionedUrns, + @Nonnull Set aspectNames) throws URISyntaxException { + return null; + } + + @Override + public EnvelopedAspect getLatestEnvelopedAspect(@Nonnull String entityName, @Nonnull Urn urn, + @Nonnull String aspectName) throws Exception { + return null; + } + + @Override + public EnvelopedAspect getEnvelopedAspect(@Nonnull String entityName, @Nonnull Urn urn, @Nonnull String aspectName, + long version) throws Exception { + return null; + } + + @Override + public VersionedAspect getVersionedAspect(@Nonnull Urn urn, @Nonnull String aspectName, long version) { + return null; + } + + @Override + public ListResult listLatestAspects(@Nonnull String entityName, @Nonnull String aspectName, int start, + int count) { + return null; + } + + @Nonnull + @Override + protected UpdateAspectResult ingestAspectToLocalDB(@Nonnull Urn urn, @Nonnull String aspectName, + @Nonnull Function, RecordTemplate> updateLambda, @Nonnull AuditStamp auditStamp, + @Nonnull SystemMetadata systemMetadata) { + return new UpdateAspectResult(UrnUtils.getUrn(DATASET_URN), null, + null, null, null, null, null, 0L); + } + + @Nonnull + @Override + protected List> ingestAspectsToLocalDB(@Nonnull Urn urn, + @Nonnull List> aspectRecordsToIngest, @Nonnull AuditStamp auditStamp, + @Nonnull SystemMetadata providedSystemMetadata) { + return Collections.emptyList(); + } + + @Override + public RecordTemplate updateAspect(@Nonnull Urn urn, @Nonnull String entityName, @Nonnull String aspectName, + @Nonnull AspectSpec aspectSpec, @Nonnull RecordTemplate newValue, @Nonnull AuditStamp auditStamp, + @Nonnull long version, @Nonnull boolean emitMae) { + return null; + } + + @Override + public ListUrnsResult listUrns(@Nonnull String entityName, int start, int count) { + return null; + } + + @Override + public void setWritable(boolean canWrite) { + + } + + @Override + public RollbackRunResult rollbackWithConditions(List aspectRows, Map conditions, + boolean hardDelete) { + return null; + } + + @Override + public RollbackRunResult deleteUrn(Urn urn) { + return null; + } + + @Override + public Boolean exists(Urn urn) { + return null; + } +} diff --git a/metadata-service/openapi-servlet/src/test/java/mock/MockEntitySpec.java b/metadata-service/openapi-servlet/src/test/java/mock/MockEntitySpec.java new file mode 100644 index 0000000000000..515c35fa878e2 --- /dev/null +++ b/metadata-service/openapi-servlet/src/test/java/mock/MockEntitySpec.java @@ -0,0 +1,118 @@ +package mock; + +import com.linkedin.common.GlossaryTerms; +import com.linkedin.common.SubTypes; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.TyperefDataSchema; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.dataset.DatasetProfile; +import com.linkedin.dataset.ViewProperties; +import com.linkedin.metadata.key.CorpUserKey; +import com.linkedin.metadata.key.DataPlatformKey; +import com.linkedin.metadata.key.DatasetKey; +import com.linkedin.metadata.key.GlossaryTermKey; +import com.linkedin.metadata.key.TagKey; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.annotation.AspectAnnotation; +import com.linkedin.metadata.models.annotation.EntityAnnotation; +import com.linkedin.schema.SchemaMetadata; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.linkedin.metadata.Constants.*; + + +public class MockEntitySpec implements EntitySpec { + + private String _name; + + public MockEntitySpec(String name) { + _name = name; + } + + @Override + public String getName() { + return _name; + } + + @Override + public EntityAnnotation getEntityAnnotation() { + return null; + } + + @Override + public String getKeyAspectName() { + return null; + } + + @Override + public AspectSpec getKeyAspectSpec() { + if (DATASET_ENTITY_NAME.equals(_name)) { + DatasetKey datasetKey = new DatasetKey(); + return createAspectSpec(datasetKey, DATASET_KEY_ASPECT_NAME); + } else if (DATA_PLATFORM_ENTITY_NAME.equals(_name)) { + DataPlatformKey dataPlatformKey = new DataPlatformKey(); + return createAspectSpec(dataPlatformKey, DATA_PLATFORM_KEY_ASPECT_NAME); + } else if (TAG_ENTITY_NAME.equals(_name)) { + TagKey tagKey = new TagKey(); + return createAspectSpec(tagKey, TAG_KEY_ASPECT_NAME); + } else if (GLOSSARY_TERM_ENTITY_NAME.equals(_name)) { + GlossaryTermKey glossaryTermKey = new GlossaryTermKey(); + return createAspectSpec(glossaryTermKey, GLOSSARY_TERM_KEY_ASPECT_NAME); + } else if (CORP_USER_ENTITY_NAME.equals(_name)) { + CorpUserKey corpUserKey = new CorpUserKey(); + return createAspectSpec(corpUserKey, CORP_USER_KEY_ASPECT_NAME); + } + return null; + } + + private AspectSpec createAspectSpec(T type, String name) { + return new MockAspectSpec(new AspectAnnotation(name, false, false, null), + Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), + Collections.emptyList(), type.schema(), (Class) type.getClass().asSubclass(RecordTemplate.class)); + } + + @Override + public List getAspectSpecs() { + return Collections.emptyList(); + } + + @Override + public Map getAspectSpecMap() { + return Collections.emptyMap(); + } + + @Override + public Boolean hasAspect(String name) { + return false; + } + + private static final Map ASPECT_TYPE_MAP; + + static { + ASPECT_TYPE_MAP = new HashMap<>(); + ASPECT_TYPE_MAP.put(DATASET_KEY_ASPECT_NAME, new DatasetKey()); + ASPECT_TYPE_MAP.put(VIEW_PROPERTIES_ASPECT_NAME, new ViewProperties()); + ASPECT_TYPE_MAP.put(SCHEMA_METADATA_ASPECT_NAME, new SchemaMetadata()); + ASPECT_TYPE_MAP.put(SUB_TYPES_ASPECT_NAME, new SubTypes()); + ASPECT_TYPE_MAP.put("datasetProfile", new DatasetProfile()); + ASPECT_TYPE_MAP.put(GLOSSARY_TERMS_ASPECT_NAME, new GlossaryTerms()); + } + @Override + public AspectSpec getAspectSpec(String name) { + return createAspectSpec(ASPECT_TYPE_MAP.get(name), name); + } + + @Override + public RecordDataSchema getSnapshotSchema() { + return null; + } + + @Override + public TyperefDataSchema getAspectTyperefSchema() { + return null; + } +}